diff options
| author | Jakub Jelinek <jakub@redhat.com> | 2023-11-15 12:45:58 +0100 |
|---|---|---|
| committer | Jakub Jelinek <jakub@redhat.com> | 2023-11-15 12:45:58 +0100 |
| commit | 28219f7f99a80519d1c6ab5e5dc83b4c7f8d7251 (patch) | |
| tree | 42e3657c58ff08a654f04aeb0f43b3bc75930bbc | |
| parent | 4d86dc51e34d2a5695b617afeb56e3414836a79a (diff) | |
libsanitizer: merge from upstream (c425db2eb558c263)
The following patch is result of libsanitizer/merge.sh
from c425db2eb558c263 (yesterday evening).
Bootstrapped/regtested on x86_64-linux and i686-linux (together with
the follow-up 3 patches I'm about to post).
BTW, seems upstream has added riscv64 support for I think lsan/tsan,
so if anyone is willing to try it there, it would be a matter of
copying e.g. the s390*-*-linux* libsanitizer/configure.tgt entry
to riscv64-*-linux* with the obvious s/s390x/riscv64/ change in it.
192 files changed, 6831 insertions, 2854 deletions
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE index ef904af5e35..2ee0843e704 100644 --- a/libsanitizer/MERGE +++ b/libsanitizer/MERGE @@ -1,4 +1,4 @@ -87e6e490e79384a523bc7f0216c3db60227d6d58 +c425db2eb558c26377edc04e062c0c1f999b2770 The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libsanitizer/asan/asan_allocator.cpp b/libsanitizer/asan/asan_allocator.cpp index 19d7777c402..22dcf613270 100644 --- a/libsanitizer/asan/asan_allocator.cpp +++ b/libsanitizer/asan/asan_allocator.cpp @@ -16,6 +16,7 @@ #include "asan_allocator.h" +#include "asan_internal.h" #include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_report.h" @@ -24,6 +25,7 @@ #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_internal_defs.h" @@ -190,28 +192,56 @@ class LargeChunkHeader { } }; +static void FillChunk(AsanChunk *m) { + // FIXME: Use ReleaseMemoryPagesToOS. + Flags &fl = *flags(); + + if (fl.max_free_fill_size > 0) { + // We have to skip the chunk header, it contains free_context_id. + uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size; + if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area. + uptr size_to_fill = m->UsedSize() - kChunkHeader2Size; + size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size); + REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill); + } + } +} + struct QuarantineCallback { QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack) : cache_(cache), stack_(stack) { } - void Recycle(AsanChunk *m) { + void PreQuarantine(AsanChunk *m) const { + FillChunk(m); + // Poison the region. + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), + kAsanHeapFreeMagic); + } + + void Recycle(AsanChunk *m) const { void *p = get_allocator().GetBlockBegin(m); - if (p != m) { - // Clear the magic value, as allocator internals may overwrite the - // contents of deallocated chunk, confusing GetAsanChunk lookup. - reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr); - } - u8 old_chunk_state = CHUNK_QUARANTINE; - if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, - CHUNK_INVALID, memory_order_acquire)) { - CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE); - } + // The secondary will immediately unpoison and unmap the memory, so this + // branch is unnecessary. + if (get_allocator().FromPrimary(p)) { + if (p != m) { + // Clear the magic value, as allocator internals may overwrite the + // contents of deallocated chunk, confusing GetAsanChunk lookup. + reinterpret_cast<LargeChunkHeader *>(p)->Set(nullptr); + } - PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), - kAsanHeapLeftRedzoneMagic); + u8 old_chunk_state = CHUNK_QUARANTINE; + if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, + CHUNK_INVALID, + memory_order_acquire)) { + CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE); + } + + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), + kAsanHeapLeftRedzoneMagic); + } // Statistics. AsanStats &thread_stats = GetCurrentThreadStats(); @@ -221,7 +251,17 @@ struct QuarantineCallback { get_allocator().Deallocate(cache_, p); } - void *Allocate(uptr size) { + void RecyclePassThrough(AsanChunk *m) const { + // Recycle for the secondary will immediately unpoison and unmap the + // memory, so quarantine preparation is unnecessary. + if (get_allocator().FromPrimary(m)) { + // The primary allocation may need pattern fill if enabled. + FillChunk(m); + } + Recycle(m); + } + + void *Allocate(uptr size) const { void *res = get_allocator().Allocate(cache_, size, 1); // TODO(alekseys): Consider making quarantine OOM-friendly. if (UNLIKELY(!res)) @@ -229,9 +269,7 @@ struct QuarantineCallback { return res; } - void Deallocate(void *p) { - get_allocator().Deallocate(cache_, p); - } + void Deallocate(void *p) const { get_allocator().Deallocate(cache_, p); } private: AllocatorCache* const cache_; @@ -248,6 +286,22 @@ void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { thread_stats.mmaps++; thread_stats.mmaped += size; } + +void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const { + uptr user_end = RoundDownTo(user_begin + user_size, ASAN_SHADOW_GRANULARITY); + user_begin = RoundUpTo(user_begin, ASAN_SHADOW_GRANULARITY); + // The secondary mapping will be immediately returned to user, no value + // poisoning that with non-zero just before unpoisoning by Allocate(). So just + // poison head/tail invisible to Allocate(). + PoisonShadow(p, user_begin - p, kAsanHeapLeftRedzoneMagic); + PoisonShadow(user_end, size - (user_end - p), kAsanHeapLeftRedzoneMagic); + // Statistics. + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mmaps++; + thread_stats.mmaped += size; +} + void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { PoisonShadow(p, size, 0); // We are about to unmap a chunk of user memory. @@ -387,8 +441,9 @@ struct Allocator { } void GetOptions(AllocatorOptions *options) const { - options->quarantine_size_mb = quarantine.GetSize() >> 20; - options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10; + options->quarantine_size_mb = quarantine.GetMaxSize() >> 20; + options->thread_local_quarantine_size_kb = + quarantine.GetMaxCacheSize() >> 10; options->min_redzone = atomic_load(&min_redzone, memory_order_acquire); options->max_redzone = atomic_load(&max_redzone, memory_order_acquire); options->may_return_null = AllocatorMayReturnNull(); @@ -472,7 +527,7 @@ struct Allocator { // -------------------- Allocation/Deallocation routines --------------- void *Allocate(uptr size, uptr alignment, BufferedStackTrace *stack, AllocType alloc_type, bool can_fill) { - if (UNLIKELY(!asan_inited)) + if (UNLIKELY(!AsanInited())) AsanInitFromRtl(); if (UNLIKELY(IsRssLimitExceeded())) { if (AllocatorMayReturnNull()) @@ -502,9 +557,10 @@ struct Allocator { uptr needed_size = rounded_size + rz_size; if (alignment > min_alignment) needed_size += alignment; + bool from_primary = PrimaryAllocator::CanAllocate(needed_size, alignment); // If we are allocating from the secondary allocator, there will be no // automatic right redzone, so add the right redzone manually. - if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) + if (!from_primary) needed_size += rz_size; CHECK(IsAligned(needed_size, min_alignment)); if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || @@ -536,15 +592,6 @@ struct Allocator { ReportOutOfMemory(size, stack); } - if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) { - // Heap poisoning is enabled, but the allocator provides an unpoisoned - // chunk. This is possible if CanPoisonMemory() was false for some - // time, for example, due to flags()->start_disabled. - // Anyway, poison the block before using it for anything else. - uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); - PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); - } - uptr alloc_beg = reinterpret_cast<uptr>(allocated); uptr alloc_end = alloc_beg + needed_size; uptr user_beg = alloc_beg + rz_size; @@ -561,6 +608,17 @@ struct Allocator { m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack)); + if (!from_primary || *(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0) { + // The allocator provides an unpoisoned chunk. This is possible for the + // secondary allocator, or if CanPoisonMemory() was false for some time, + // for example, due to flags()->start_disabled. Anyway, poison left and + // right of the block before using it for anything else. + uptr tail_beg = RoundUpTo(user_end, ASAN_SHADOW_GRANULARITY); + uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(allocated); + PoisonShadow(alloc_beg, user_beg - alloc_beg, kAsanHeapLeftRedzoneMagic); + PoisonShadow(tail_beg, tail_end - tail_beg, kAsanHeapLeftRedzoneMagic); + } + uptr size_rounded_down_to_granularity = RoundDownTo(size, ASAN_SHADOW_GRANULARITY); // Unpoison the bulk of the memory region. @@ -628,25 +686,6 @@ struct Allocator { AsanThread *t = GetCurrentThread(); m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack)); - Flags &fl = *flags(); - if (fl.max_free_fill_size > 0) { - // We have to skip the chunk header, it contains free_context_id. - uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size; - if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area. - uptr size_to_fill = m->UsedSize() - kChunkHeader2Size; - size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size); - REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill); - } - } - - // Poison the region. - PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), - kAsanHeapFreeMagic); - - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.frees++; - thread_stats.freed += m->UsedSize(); - // Push into quarantine. if (t) { AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); @@ -699,6 +738,10 @@ struct Allocator { } } + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.frees++; + thread_stats.freed += m->UsedSize(); + QuarantineChunk(m, ptr, stack); } @@ -798,6 +841,10 @@ struct Allocator { return m->UsedSize(); } + uptr AllocationSizeFast(uptr p) { + return reinterpret_cast<AsanChunk *>(p - kChunkHeaderSize)->UsedSize(); + } + AsanChunkView FindHeapChunkByAddress(uptr addr) { AsanChunk *m1 = GetAsanChunkByAddr(addr); sptr offset = 0; @@ -1198,6 +1245,13 @@ uptr __sanitizer_get_allocated_size(const void *p) { return allocated_size; } +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = instance.AllocationSizeFast(reinterpret_cast<uptr>(p)); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + const void *__sanitizer_get_allocated_begin(const void *p) { return AllocationBegin(p); } diff --git a/libsanitizer/asan/asan_allocator.h b/libsanitizer/asan/asan_allocator.h index 6a12a6c6025..c3c4fae85b1 100644 --- a/libsanitizer/asan/asan_allocator.h +++ b/libsanitizer/asan/asan_allocator.h @@ -114,32 +114,98 @@ class AsanChunkFifoList: public IntrusiveList<AsanChunk> { struct AsanMapUnmapCallback { void OnMap(uptr p, uptr size) const; + void OnMapSecondary(uptr p, uptr size, uptr user_begin, uptr user_size) const; void OnUnmap(uptr p, uptr size) const; }; #if SANITIZER_CAN_USE_ALLOCATOR64 # if SANITIZER_FUCHSIA +// This is a sentinel indicating we do not want the primary allocator arena to +// be placed at a fixed address. It will be anonymously mmap'd. const uptr kAllocatorSpace = ~(uptr)0; -const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +# if SANITIZER_RISCV64 + +// These are sanitizer tunings that allow all bringup tests for RISCV-64 Sv39 + +// Fuchsia to run with asan-instrumented. That is, we can run bringup, e2e, +// libc, and scudo tests with this configuration. +// +// TODO: This is specifically tuned for Sv39. 48/57 will likely require other +// tunings, or possibly use the same tunings Fuchsia uses for other archs. The +// VMA size isn't technically tied to the Fuchsia System ABI, so once 48/57 is +// supported, we'd need a way of dynamically checking what the VMA size is and +// determining optimal configuration. + +// This indicates the total amount of space dedicated for the primary allocator +// during initialization. This is roughly proportional to the size set by the +// FuchsiaConfig for scudo (~11.25GB == ~2^33.49). Requesting any more could +// lead to some failures in sanitized bringup tests where we can't allocate new +// vmars because there wouldn't be enough contiguous space. We could try 2^34 if +// we re-evaluate the SizeClassMap settings. +const uptr kAllocatorSize = UINT64_C(1) << 33; // 8GB + +// This is roughly equivalent to the configuration for the VeryDenseSizeClassMap +// but has fewer size classes (ideally at most 32). Fewer class sizes means the +// region size for each class is larger, thus less chances of running out of +// space for each region. The main differences are the MidSizeLog (which is +// smaller) and the MaxSizeLog (which is larger). +// +// - The MaxSizeLog is higher to allow some of the largest allocations I've +// observed to be placed in the primary allocator's arena as opposed to being +// mmap'd by the secondary allocator. This helps reduce fragmentation from +// large classes. A huge example of this the scudo allocator tests (and its +// testing infrastructure) which malloc's/new's objects on the order of +// hundreds of kilobytes which normally would not be in the primary allocator +// arena with the default VeryDenseSizeClassMap. +// - The MidSizeLog is reduced to help shrink the number of size classes and +// increase region size. Without this, we'd see ASan complain many times about +// a region running out of available space. +// +// This differs a bit from the fuchsia config in scudo, mainly from the NumBits, +// MaxSizeLog, and NumCachedHintT. This should place the number of size classes +// for scudo at 45 and some large objects allocated by this config would be +// placed in the arena whereas scudo would mmap them. The asan allocator needs +// to have a number of classes that are a power of 2 for various internal things +// to work, so we can't match the scudo settings to a tee. The sanitizer +// allocator is slightly slower than scudo's but this is enough to get +// memory-intensive scudo tests to run with asan instrumentation. +typedef SizeClassMap</*kNumBits=*/2, + /*kMinSizeLog=*/5, + /*kMidSizeLog=*/8, + /*kMaxSizeLog=*/18, + /*kNumCachedHintT=*/8, + /*kMaxBytesCachedLog=*/10> + SizeClassMap; +static_assert(SizeClassMap::kNumClassesRounded <= 32, + "The above tunings were specifically selected to ensure there " + "would be at most 32 size classes. This restriction could be " + "loosened to 64 size classes if we can find a configuration of " + "allocator size and SizeClassMap tunings that allows us to " + "reliably run all bringup tests in a sanitized environment."); + +# else +// These are the default allocator tunings for non-RISCV environments where the +// VMA is usually 48 bits and we have lots of space. +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef DefaultSizeClassMap SizeClassMap; -# elif defined(__powerpc64__) +# endif +# elif defined(__powerpc64__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. typedef DefaultSizeClassMap SizeClassMap; -# elif defined(__aarch64__) && SANITIZER_ANDROID +# elif defined(__aarch64__) && SANITIZER_ANDROID // Android needs to support 39, 42 and 48 bit VMA. const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x2000000000ULL; // 128G. typedef VeryCompactSizeClassMap SizeClassMap; -#elif SANITIZER_RISCV64 +# elif SANITIZER_RISCV64 const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x2000000000ULL; // 128G. typedef VeryDenseSizeClassMap SizeClassMap; -#elif defined(__sparc__) +# elif defined(__sparc__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x20000000000ULL; // 2T. typedef DefaultSizeClassMap SizeClassMap; -# elif SANITIZER_WINDOWS +# elif SANITIZER_WINDOWS const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x8000000000ULL; // 500G typedef DefaultSizeClassMap SizeClassMap; diff --git a/libsanitizer/asan/asan_descriptions.cpp b/libsanitizer/asan/asan_descriptions.cpp index fbe92572b55..ef6f3e0a096 100644 --- a/libsanitizer/asan/asan_descriptions.cpp +++ b/libsanitizer/asan/asan_descriptions.cpp @@ -49,14 +49,14 @@ void DescribeThread(AsanThreadContext *context) { } context->announced = true; InternalScopedString str; - str.append("Thread %s", AsanThreadIdAndName(context).c_str()); + str.AppendF("Thread %s", AsanThreadIdAndName(context).c_str()); if (context->parent_tid == kInvalidTid) { - str.append(" created by unknown thread\n"); + str.Append(" created by unknown thread\n"); Printf("%s", str.data()); return; } - str.append(" created by %s here:\n", - AsanThreadIdAndName(context->parent_tid).c_str()); + str.AppendF(" created by %s here:\n", + AsanThreadIdAndName(context->parent_tid).c_str()); Printf("%s", str.data()); StackDepotGet(context->stack_id).Print(); // Recursively described parent thread if needed. @@ -126,29 +126,29 @@ static void GetAccessToHeapChunkInformation(ChunkAccess *descr, static void PrintHeapChunkAccess(uptr addr, const ChunkAccess &descr) { Decorator d; InternalScopedString str; - str.append("%s", d.Location()); + str.Append(d.Location()); switch (descr.access_type) { case kAccessTypeLeft: - str.append("%p is located %zd bytes before", - (void *)descr.bad_addr, descr.offset); + str.AppendF("%p is located %zd bytes before", (void *)descr.bad_addr, + descr.offset); break; case kAccessTypeRight: - str.append("%p is located %zd bytes after", - (void *)descr.bad_addr, descr.offset); + str.AppendF("%p is located %zd bytes after", (void *)descr.bad_addr, + descr.offset); break; case kAccessTypeInside: - str.append("%p is located %zd bytes inside of", (void *)descr.bad_addr, - descr.offset); + str.AppendF("%p is located %zd bytes inside of", (void *)descr.bad_addr, + descr.offset); break; case kAccessTypeUnknown: - str.append( + str.AppendF( "%p is located somewhere around (this is AddressSanitizer bug!)", (void *)descr.bad_addr); } - str.append(" %zu-byte region [%p,%p)\n", descr.chunk_size, - (void *)descr.chunk_begin, - (void *)(descr.chunk_begin + descr.chunk_size)); - str.append("%s", d.Default()); + str.AppendF(" %zu-byte region [%p,%p)\n", descr.chunk_size, + (void *)descr.chunk_begin, + (void *)(descr.chunk_begin + descr.chunk_size)); + str.Append(d.Default()); Printf("%s", str.data()); } @@ -243,24 +243,24 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, pos_descr = "underflows"; } InternalScopedString str; - str.append(" [%zd, %zd)", var.beg, var_end); + str.AppendF(" [%zd, %zd)", var.beg, var_end); // Render variable name. - str.append(" '"); + str.AppendF(" '"); for (uptr i = 0; i < var.name_len; ++i) { - str.append("%c", var.name_pos[i]); + str.AppendF("%c", var.name_pos[i]); } - str.append("'"); + str.AppendF("'"); if (var.line > 0) { - str.append(" (line %zd)", var.line); + str.AppendF(" (line %zd)", var.line); } if (pos_descr) { Decorator d; // FIXME: we may want to also print the size of the access here, // but in case of accesses generated by memset it may be confusing. - str.append("%s <== Memory access at offset %zd %s this variable%s\n", - d.Location(), addr, pos_descr, d.Default()); + str.AppendF("%s <== Memory access at offset %zd %s this variable%s\n", + d.Location(), addr, pos_descr, d.Default()); } else { - str.append("\n"); + str.AppendF("\n"); } Printf("%s", str.data()); } @@ -277,23 +277,23 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size, const __asan_global &g) { InternalScopedString str; Decorator d; - str.append("%s", d.Location()); + str.Append(d.Location()); if (addr < g.beg) { - str.append("%p is located %zd bytes before", (void *)addr, - g.beg - addr); + str.AppendF("%p is located %zd bytes before", (void *)addr, g.beg - addr); } else if (addr + access_size > g.beg + g.size) { if (addr < g.beg + g.size) addr = g.beg + g.size; - str.append("%p is located %zd bytes after", (void *)addr, - addr - (g.beg + g.size)); + str.AppendF("%p is located %zd bytes after", (void *)addr, + addr - (g.beg + g.size)); } else { // Can it happen? - str.append("%p is located %zd bytes inside of", (void *)addr, addr - g.beg); + str.AppendF("%p is located %zd bytes inside of", (void *)addr, + addr - g.beg); } - str.append(" global variable '%s' defined in '", - MaybeDemangleGlobalName(g.name)); - PrintGlobalLocation(&str, g); - str.append("' (0x%zx) of size %zu\n", g.beg, g.size); - str.append("%s", d.Default()); + str.AppendF(" global variable '%s' defined in '", + MaybeDemangleGlobalName(g.name)); + PrintGlobalLocation(&str, g, /*print_module_name=*/false); + str.AppendF("' (0x%zx) of size %zu\n", g.beg, g.size); + str.Append(d.Default()); PrintGlobalNameIfASCII(&str, g); Printf("%s", str.data()); } diff --git a/libsanitizer/asan/asan_errors.cpp b/libsanitizer/asan/asan_errors.cpp index cc8dc26f5b7..3f2d13e3146 100644 --- a/libsanitizer/asan/asan_errors.cpp +++ b/libsanitizer/asan/asan_errors.cpp @@ -362,8 +362,8 @@ void ErrorODRViolation::Print() { Printf("%s", d.Default()); InternalScopedString g1_loc; InternalScopedString g2_loc; - PrintGlobalLocation(&g1_loc, global1); - PrintGlobalLocation(&g2_loc, global2); + PrintGlobalLocation(&g1_loc, global1, /*print_module_name=*/true); + PrintGlobalLocation(&g2_loc, global2, /*print_module_name=*/true); Printf(" [1] size=%zd '%s' %s\n", global1.size, MaybeDemangleGlobalName(global1.name), g1_loc.data()); Printf(" [2] size=%zd '%s' %s\n", global2.size, @@ -379,8 +379,8 @@ void ErrorODRViolation::Print() { "HINT: if you don't care about these errors you may set " "ASAN_OPTIONS=detect_odr_violation=0\n"); InternalScopedString error_msg; - error_msg.append("%s: global '%s' at %s", scariness.GetDescription(), - MaybeDemangleGlobalName(global1.name), g1_loc.data()); + error_msg.AppendF("%s: global '%s' at %s", scariness.GetDescription(), + MaybeDemangleGlobalName(global1.name), g1_loc.data()); ReportErrorSummary(error_msg.data()); } @@ -517,15 +517,15 @@ static void PrintShadowByte(InternalScopedString *str, const char *before, } static void PrintLegend(InternalScopedString *str) { - str->append( + str->AppendF( "Shadow byte legend (one shadow byte represents %d " "application bytes):\n", (int)ASAN_SHADOW_GRANULARITY); PrintShadowByte(str, " Addressable: ", 0); - str->append(" Partially addressable: "); + str->AppendF(" Partially addressable: "); for (u8 i = 1; i < ASAN_SHADOW_GRANULARITY; i++) PrintShadowByte(str, "", i, " "); - str->append("\n"); + str->AppendF("\n"); PrintShadowByte(str, " Heap left redzone: ", kAsanHeapLeftRedzoneMagic); PrintShadowByte(str, " Freed heap region: ", kAsanHeapFreeMagic); @@ -559,8 +559,8 @@ static void PrintShadowBytes(InternalScopedString *str, const char *before, u8 *bytes, u8 *guilty, uptr n) { Decorator d; if (before) - str->append("%s%p:", before, - (void *)ShadowToMem(reinterpret_cast<uptr>(bytes))); + str->AppendF("%s%p:", before, + (void *)ShadowToMem(reinterpret_cast<uptr>(bytes))); for (uptr i = 0; i < n; i++) { u8 *p = bytes + i; const char *before = @@ -568,7 +568,7 @@ static void PrintShadowBytes(InternalScopedString *str, const char *before, const char *after = p == guilty ? "]" : ""; PrintShadowByte(str, before, *p, after); } - str->append("\n"); + str->AppendF("\n"); } static void PrintShadowMemoryForAddress(uptr addr) { @@ -577,7 +577,7 @@ static void PrintShadowMemoryForAddress(uptr addr) { const uptr n_bytes_per_row = 16; uptr aligned_shadow = shadow_addr & ~(n_bytes_per_row - 1); InternalScopedString str; - str.append("Shadow bytes around the buggy address:\n"); + str.AppendF("Shadow bytes around the buggy address:\n"); for (int i = -5; i <= 5; i++) { uptr row_shadow_addr = aligned_shadow + i * n_bytes_per_row; // Skip rows that would be outside the shadow range. This can happen when diff --git a/libsanitizer/asan/asan_fake_stack.cpp b/libsanitizer/asan/asan_fake_stack.cpp index 74a039b6579..7443ff16698 100644 --- a/libsanitizer/asan/asan_fake_stack.cpp +++ b/libsanitizer/asan/asan_fake_stack.cpp @@ -68,8 +68,8 @@ void FakeStack::Destroy(int tid) { if (Verbosity() >= 2) { InternalScopedString str; for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++) - str.append("%zd: %zd/%zd; ", class_id, hint_position_[class_id], - NumberOfFrames(stack_size_log(), class_id)); + str.AppendF("%zd: %zd/%zd; ", class_id, hint_position_[class_id], + NumberOfFrames(stack_size_log(), class_id)); Report("T%d: FakeStack destroyed: %s\n", tid, str.data()); } uptr size = RequiredSize(stack_size_log_); @@ -133,6 +133,12 @@ void FakeStack::HandleNoReturn() { needs_gc_ = true; } +// Hack: The statement below is not true if we take into account sigaltstack or +// makecontext. It should be possible to make GC to discard wrong stack frame if +// we use these tools. For now, let's support the simplest case and allow GC to +// discard only frames from the default stack, assuming there is no buffer on +// the stack which is used for makecontext or sigaltstack. +// // When throw, longjmp or some such happens we don't call OnFree() and // as the result may leak one or more fake frames, but the good news is that // we are notified about all such events by HandleNoReturn(). @@ -140,6 +146,14 @@ void FakeStack::HandleNoReturn() { // We do it based on their 'real_stack' values -- everything that is lower // than the current real_stack is garbage. NOINLINE void FakeStack::GC(uptr real_stack) { + AsanThread *curr_thread = GetCurrentThread(); + if (!curr_thread) + return; // Try again when we have a thread. + auto top = curr_thread->stack_top(); + auto bottom = curr_thread->stack_bottom(); + if (real_stack < bottom || real_stack > top) + return; // Not the default stack. + for (uptr class_id = 0; class_id < kNumberOfSizeClasses; class_id++) { u8 *flags = GetFlags(stack_size_log(), class_id); for (uptr i = 0, n = NumberOfFrames(stack_size_log(), class_id); i < n; @@ -147,8 +161,12 @@ NOINLINE void FakeStack::GC(uptr real_stack) { if (flags[i] == 0) continue; // not allocated. FakeFrame *ff = reinterpret_cast<FakeFrame *>( GetFrame(stack_size_log(), class_id, i)); - if (ff->real_stack < real_stack) { + // GC only on the default stack. + if (bottom < ff->real_stack && ff->real_stack < real_stack) { flags[i] = 0; + // Poison the frame, so the any access will be reported as UAR. + SetShadow(reinterpret_cast<uptr>(ff), BytesInSizeClass(class_id), + class_id, kMagic8); } } } @@ -205,11 +223,12 @@ static FakeStack *GetFakeStackFastAlways() { static ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) { FakeStack *fs = GetFakeStackFast(); - if (!fs) return 0; - uptr local_stack; - uptr real_stack = reinterpret_cast<uptr>(&local_stack); - FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack); - if (!ff) return 0; // Out of fake stack. + if (!fs) + return 0; + FakeFrame *ff = + fs->Allocate(fs->stack_size_log(), class_id, GET_CURRENT_FRAME()); + if (!ff) + return 0; // Out of fake stack. uptr ptr = reinterpret_cast<uptr>(ff); SetShadow(ptr, size, class_id, 0); return ptr; @@ -219,9 +238,8 @@ static ALWAYS_INLINE uptr OnMallocAlways(uptr class_id, uptr size) { FakeStack *fs = GetFakeStackFastAlways(); if (!fs) return 0; - uptr local_stack; - uptr real_stack = reinterpret_cast<uptr>(&local_stack); - FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack); + FakeFrame *ff = + fs->Allocate(fs->stack_size_log(), class_id, GET_CURRENT_FRAME()); if (!ff) return 0; // Out of fake stack. uptr ptr = reinterpret_cast<uptr>(ff); diff --git a/libsanitizer/asan/asan_globals.cpp b/libsanitizer/asan/asan_globals.cpp index 01a243927ca..6ac64c4b776 100644 --- a/libsanitizer/asan/asan_globals.cpp +++ b/libsanitizer/asan/asan_globals.cpp @@ -36,7 +36,6 @@ struct ListOfGlobals { }; static Mutex mu_for_globals; -static LowLevelAllocator allocator_for_globals; static ListOfGlobals *list_of_all_globals; static const int kDynamicInitGlobalsInitialCapacity = 512; @@ -81,18 +80,19 @@ static bool IsAddressNearGlobal(uptr addr, const __asan_global &g) { } static void ReportGlobal(const Global &g, const char *prefix) { + DataInfo info; + bool symbolized = Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info); Report( - "%s Global[%p]: beg=%p size=%zu/%zu name=%s module=%s dyn_init=%zu " + "%s Global[%p]: beg=%p size=%zu/%zu name=%s source=%s module=%s " + "dyn_init=%zu " "odr_indicator=%p\n", prefix, (void *)&g, (void *)g.beg, g.size, g.size_with_redzone, g.name, - g.module_name, g.has_dynamic_init, (void *)g.odr_indicator); + g.module_name, (symbolized ? info.module : "?"), g.has_dynamic_init, + (void *)g.odr_indicator); - DataInfo info; - Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info); - if (info.line != 0) { + if (symbolized && info.line != 0) { Report(" location: name=%s, %d\n", info.file, static_cast<int>(info.line)); - } - else if (g.gcc_location != 0) { + } else if (g.gcc_location != 0) { // Fallback to Global::gcc_location Report(" location: name=%s, %d\n", g.gcc_location->filename, g.gcc_location->line_no); } @@ -158,6 +158,23 @@ static void CheckODRViolationViaIndicator(const Global *g) { } } +// Check ODR violation for given global G by checking if it's already poisoned. +// We use this method in case compiler doesn't use private aliases for global +// variables. +static void CheckODRViolationViaPoisoning(const Global *g) { + if (__asan_region_is_poisoned(g->beg, g->size_with_redzone)) { + // This check may not be enough: if the first global is much larger + // the entire redzone of the second global may be within the first global. + for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) { + if (g->beg == l->g->beg && + (flags()->detect_odr_violation >= 2 || g->size != l->g->size) && + !IsODRViolationSuppressed(g->name)) + ReportODRViolation(g, FindRegistrationSite(g), + l->g, FindRegistrationSite(l->g)); + } + } +} + // Clang provides two different ways for global variables protection: // it can poison the global itself or its private alias. In former // case we may poison same symbol multiple times, that can help us to @@ -182,7 +199,7 @@ static inline bool UseODRIndicator(const Global *g) { // This function may be called more than once for every global // so we store the globals in a map. static void RegisterGlobal(const Global *g) { - CHECK(asan_inited); + CHECK(AsanInited()); if (flags()->report_globals >= 2) ReportGlobal(*g, "Added"); CHECK(flags()->report_globals); @@ -203,16 +220,18 @@ static void RegisterGlobal(const Global *g) { // where two globals with the same name are defined in different modules. if (UseODRIndicator(g)) CheckODRViolationViaIndicator(g); + else + CheckODRViolationViaPoisoning(g); } if (CanPoisonMemory()) PoisonRedZones(*g); - ListOfGlobals *l = new(allocator_for_globals) ListOfGlobals; + ListOfGlobals *l = new (GetGlobalLowLevelAllocator()) ListOfGlobals; l->g = g; l->next = list_of_all_globals; list_of_all_globals = l; if (g->has_dynamic_init) { if (!dynamic_init_globals) { - dynamic_init_globals = new (allocator_for_globals) VectorOfGlobals; + dynamic_init_globals = new (GetGlobalLowLevelAllocator()) VectorOfGlobals; dynamic_init_globals->reserve(kDynamicInitGlobalsInitialCapacity); } DynInitGlobal dyn_global = { *g, false }; @@ -221,7 +240,7 @@ static void RegisterGlobal(const Global *g) { } static void UnregisterGlobal(const Global *g) { - CHECK(asan_inited); + CHECK(AsanInited()); if (flags()->report_globals >= 2) ReportGlobal(*g, "Removed"); CHECK(flags()->report_globals); @@ -277,24 +296,28 @@ void PrintGlobalNameIfASCII(InternalScopedString *str, const __asan_global &g) { if (c == '\0' || !IsASCII(c)) return; } if (*(char *)(g.beg + g.size - 1) != '\0') return; - str->append(" '%s' is ascii string '%s'\n", MaybeDemangleGlobalName(g.name), - (char *)g.beg); + str->AppendF(" '%s' is ascii string '%s'\n", MaybeDemangleGlobalName(g.name), + (char *)g.beg); } -void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g) { +void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g, + bool print_module_name) { DataInfo info; - Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info); - - if (info.line != 0) { - str->append("%s:%d", info.file, static_cast<int>(info.line)); + if (Symbolizer::GetOrInit()->SymbolizeData(g.beg, &info) && info.line != 0) { + str->AppendF("%s:%d", info.file, static_cast<int>(info.line)); } else if (g.gcc_location != 0) { // Fallback to Global::gcc_location - str->append("%s", g.gcc_location->filename ? g.gcc_location->filename : g.module_name); - if (g.gcc_location->line_no) str->append(":%d", g.gcc_location->line_no); - if (g.gcc_location->column_no) str->append(":%d", g.gcc_location->column_no); + str->AppendF("%s", g.gcc_location->filename ? g.gcc_location->filename + : g.module_name); + if (g.gcc_location->line_no) + str->AppendF(":%d", g.gcc_location->line_no); + if (g.gcc_location->column_no) + str->AppendF(":%d", g.gcc_location->column_no); } else { - str->append("%s", g.module_name); + str->AppendF("%s", g.module_name); } + if (print_module_name && info.module) + str->AppendF(" in %s", info.module); } } // namespace __asan @@ -348,7 +371,7 @@ void __asan_register_globals(__asan_global *globals, uptr n) { Lock lock(&mu_for_globals); if (!global_registration_site_vector) { global_registration_site_vector = - new (allocator_for_globals) GlobalRegistrationSiteVector; + new (GetGlobalLowLevelAllocator()) GlobalRegistrationSiteVector; global_registration_site_vector->reserve(128); } GlobalRegistrationSite site = {stack_id, &globals[0], &globals[n - 1]}; @@ -410,7 +433,7 @@ void __asan_before_dynamic_init(const char *module_name) { return; bool strict_init_order = flags()->strict_init_order; CHECK(module_name); - CHECK(asan_inited); + CHECK(AsanInited()); Lock lock(&mu_for_globals); if (flags()->report_globals >= 3) Printf("DynInitPoison module: %s\n", module_name); @@ -434,7 +457,7 @@ void __asan_after_dynamic_init() { !CanPoisonMemory() || !dynamic_init_globals) return; - CHECK(asan_inited); + CHECK(AsanInited()); Lock lock(&mu_for_globals); // FIXME: Optionally report that we're unpoisoning globals from a module. for (uptr i = 0, n = dynamic_init_globals->size(); i < n; ++i) { diff --git a/libsanitizer/asan/asan_interceptors.cpp b/libsanitizer/asan/asan_interceptors.cpp index 0f160055248..234b18bd83a 100644 --- a/libsanitizer/asan/asan_interceptors.cpp +++ b/libsanitizer/asan/asan_interceptors.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "asan_interceptors.h" + #include "asan_allocator.h" #include "asan_internal.h" #include "asan_mapping.h" @@ -20,7 +21,10 @@ #include "asan_stack.h" #include "asan_stats.h" #include "asan_suppressions.h" +#include "asan_thread.h" #include "lsan/lsan_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" // There is no general interception at all on Fuchsia. @@ -84,12 +88,6 @@ using namespace __asan; DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) -#define ASAN_INTERCEPTOR_ENTER(ctx, func) \ - AsanInterceptorContext _ctx = {#func}; \ - ctx = (void *)&_ctx; \ - (void) ctx; \ - -#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ ASAN_INTERCEPT_FUNC_VER(name, ver) #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ @@ -98,15 +96,15 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) ASAN_WRITE_RANGE(ctx, ptr, size) #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ ASAN_READ_RANGE(ctx, ptr, size) -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - ASAN_INTERCEPTOR_ENTER(ctx, func); \ - do { \ - if (asan_init_is_running) \ - return REAL(func)(__VA_ARGS__); \ - if (SANITIZER_APPLE && UNLIKELY(!asan_inited)) \ - return REAL(func)(__VA_ARGS__); \ - ENSURE_ASAN_INITED(); \ - } while (false) +# define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + ASAN_INTERCEPTOR_ENTER(ctx, func); \ + do { \ + if (AsanInitIsRunning()) \ + return REAL(func)(__VA_ARGS__); \ + if (SANITIZER_APPLE && UNLIKELY(!AsanInited())) \ + return REAL(func)(__VA_ARGS__); \ + ENSURE_ASAN_INITED(); \ + } while (false) #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ do { \ } while (false) @@ -140,7 +138,7 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) # define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit() # define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) # define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() -# define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!asan_inited) +# define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!AsanInited()) # define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ if (AsanThread *t = GetCurrentThread()) { \ *begin = t->tls_begin(); \ @@ -149,22 +147,46 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) *begin = *end = 0; \ } -#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memmove); \ - ASAN_MEMMOVE_IMPL(ctx, to, from, size); \ - } while (false) +template <class Mmap> +static void* mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length, + int prot, int flags, int fd, OFF64_T offset) { + void *res = real_mmap(addr, length, prot, flags, fd, offset); + if (length && res != (void *)-1) { + const uptr beg = reinterpret_cast<uptr>(res); + DCHECK(IsAligned(beg, GetPageSize())); + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + // Only unpoison shadow if it's an ASAN managed address. + if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1)) + PoisonShadow(beg, RoundUpTo(length, GetPageSize()), 0); + } + return res; +} -#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \ - ASAN_MEMCPY_IMPL(ctx, to, from, size); \ +template <class Munmap> +static int munmap_interceptor(Munmap real_munmap, void *addr, SIZE_T length) { + // We should not tag if munmap fail, but it's to late to tag after + // real_munmap, as the pages could be mmaped by another thread. + const uptr beg = reinterpret_cast<uptr>(addr); + if (length && IsAligned(beg, GetPageSize())) { + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + // Protect from unmapping the shadow. + if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1)) + PoisonShadow(beg, rounded_length, 0); + } + return real_munmap(addr, length); +} + +# define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, \ + fd, offset) \ + do { \ + (void)(ctx); \ + return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off); \ } while (false) -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memset); \ - ASAN_MEMSET_IMPL(ctx, block, c, size); \ +# define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, length) \ + do { \ + (void)(ctx); \ + return munmap_interceptor(REAL(munmap), addr, sz); \ } while (false) #if CAN_SANITIZE_LEAKS @@ -172,6 +194,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) __lsan::ScopedInterceptorDisabler disabler #endif +#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_ASAN_INITED() + #include "sanitizer_common/sanitizer_common_interceptors.inc" #include "sanitizer_common/sanitizer_signal_interceptors.inc" @@ -196,23 +220,44 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { AsanThread *t = (AsanThread *)arg; SetCurrentThread(t); - return t->ThreadStart(GetTid()); + auto self = GetThreadSelf(); + auto args = asanThreadArgRetval().GetArgs(self); + t->ThreadStart(GetTid()); + +# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + __sanitizer_sigset_t sigset; + t->GetStartData(sigset); + SetSigProcMask(&sigset, nullptr); +# endif + + thread_return_t retval = (*args.routine)(args.arg_retval); + asanThreadArgRetval().Finish(self, retval); + return retval; } -INTERCEPTOR(int, pthread_create, void *thread, - void *attr, void *(*start_routine)(void*), void *arg) { +INTERCEPTOR(int, pthread_create, void *thread, void *attr, + void *(*start_routine)(void *), void *arg) { EnsureMainThreadIDIsCorrect(); // Strict init-order checking is thread-hostile. if (flags()->strict_init_order) StopInitOrderChecking(); GET_STACK_TRACE_THREAD; - int detached = 0; - if (attr) - REAL(pthread_attr_getdetachstate)(attr, &detached); + bool detached = [attr]() { + int d = 0; + return attr && !REAL(pthread_attr_getdetachstate)(attr, &d) && + IsStateDetached(d); + }(); u32 current_tid = GetCurrentTidOrInvalid(); - AsanThread *t = - AsanThread::Create(start_routine, arg, current_tid, &stack, detached); + + __sanitizer_sigset_t sigset = {}; +# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + ScopedBlockSignals block(&sigset); +# endif + + AsanThread *t = AsanThread::Create(sigset, current_tid, &stack, detached); int result; { @@ -220,10 +265,13 @@ INTERCEPTOR(int, pthread_create, void *thread, // stored by pthread for future reuse even after thread destruction, and // the linked list it's stored in doesn't even hold valid pointers to the // objects, the latter are calculated by obscure pointer arithmetic. -#if CAN_SANITIZE_LEAKS +# if CAN_SANITIZE_LEAKS __lsan::ScopedInterceptorDisabler disabler; -#endif - result = REAL(pthread_create)(thread, attr, asan_thread_start, t); +# endif + asanThreadArgRetval().Create(detached, {start_routine, arg}, [&]() -> uptr { + result = REAL(pthread_create)(thread, attr, asan_thread_start, t); + return result ? 0 : *(uptr *)(thread); + }); } if (result != 0) { // If the thread didn't start delete the AsanThread to avoid leaking it. @@ -234,9 +282,51 @@ INTERCEPTOR(int, pthread_create, void *thread, return result; } -INTERCEPTOR(int, pthread_join, void *t, void **arg) { - return real_pthread_join(t, arg); +INTERCEPTOR(int, pthread_join, void *thread, void **retval) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_join)(thread, retval); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_detach, void *thread) { + int result; + asanThreadArgRetval().Detach((uptr)thread, [&]() { + result = REAL(pthread_detach)(thread); + return !result; + }); + return result; +} + +INTERCEPTOR(void, pthread_exit, void *retval) { + asanThreadArgRetval().Finish(GetThreadSelf(), retval); + REAL(pthread_exit)(retval); +} + +# if ASAN_INTERCEPT_TRYJOIN +INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_tryjoin_np)(thread, ret); + return !result; + }); + return result; } +# endif + +# if ASAN_INTERCEPT_TIMEDJOIN +INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, + const struct timespec *abstime) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_timedjoin_np)(thread, ret, abstime); + return !result; + }); + return result; +} +# endif DEFINE_REAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE @@ -388,7 +478,7 @@ INTERCEPTOR(_Unwind_Reason_Code, _Unwind_SjLj_RaiseException, #if ASAN_INTERCEPT_INDEX # if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX INTERCEPTOR(char*, index, const char *string, int c) - ALIAS(WRAPPER_NAME(strchr)); + ALIAS(WRAP(strchr)); # else # if SANITIZER_APPLE DECLARE_REAL(char*, index, const char *string, int c) @@ -445,12 +535,12 @@ INTERCEPTOR(char *, strcpy, char *to, const char *from) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strcpy); #if SANITIZER_APPLE - if (UNLIKELY(!asan_inited)) + if (UNLIKELY(!AsanInited())) return REAL(strcpy)(to, from); #endif // strcpy is called from malloc_default_purgeable_zone() // in __asan::ReplaceSystemAlloc() on Mac. - if (asan_init_is_running) { + if (AsanInitIsRunning()) { return REAL(strcpy)(to, from); } ENSURE_ASAN_INITED(); @@ -466,7 +556,8 @@ INTERCEPTOR(char *, strcpy, char *to, const char *from) { INTERCEPTOR(char*, strdup, const char *s) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strdup); - if (UNLIKELY(!asan_inited)) return internal_strdup(s); + if (UNLIKELY(!AsanInited())) + return internal_strdup(s); ENSURE_ASAN_INITED(); uptr length = internal_strlen(s); if (flags()->replace_str) { @@ -484,7 +575,8 @@ INTERCEPTOR(char*, strdup, const char *s) { INTERCEPTOR(char*, __strdup, const char *s) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strdup); - if (UNLIKELY(!asan_inited)) return internal_strdup(s); + if (UNLIKELY(!AsanInited())) + return internal_strdup(s); ENSURE_ASAN_INITED(); uptr length = internal_strlen(s); if (flags()->replace_str) { @@ -512,25 +604,41 @@ INTERCEPTOR(char*, strncpy, char *to, const char *from, uptr size) { return REAL(strncpy)(to, from, size); } -INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) { - void *ctx; - ASAN_INTERCEPTOR_ENTER(ctx, strtol); - ENSURE_ASAN_INITED(); - if (!flags()->replace_str) { - return REAL(strtol)(nptr, endptr, base); - } +template <typename Fn> +static ALWAYS_INLINE auto StrtolImpl(void *ctx, Fn real, const char *nptr, + char **endptr, int base) + -> decltype(real(nullptr, nullptr, 0)) { + if (!flags()->replace_str) + return real(nptr, endptr, base); char *real_endptr; - long result = REAL(strtol)(nptr, &real_endptr, base); + auto res = real(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return result; + return res; } +# define INTERCEPTOR_STRTO_BASE(ret_type, func) \ + INTERCEPTOR(ret_type, func, const char *nptr, char **endptr, int base) { \ + void *ctx; \ + ASAN_INTERCEPTOR_ENTER(ctx, func); \ + ENSURE_ASAN_INITED(); \ + return StrtolImpl(ctx, REAL(func), nptr, endptr, base); \ + } + +INTERCEPTOR_STRTO_BASE(long, strtol) +INTERCEPTOR_STRTO_BASE(long long, strtoll) + +# if SANITIZER_GLIBC +INTERCEPTOR_STRTO_BASE(long, __isoc23_strtol) +INTERCEPTOR_STRTO_BASE(long long, __isoc23_strtoll) +# endif + INTERCEPTOR(int, atoi, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atoi); #if SANITIZER_APPLE - if (UNLIKELY(!asan_inited)) return REAL(atoi)(nptr); -#endif + if (UNLIKELY(!AsanInited())) + return REAL(atoi)(nptr); +# endif ENSURE_ASAN_INITED(); if (!flags()->replace_str) { return REAL(atoi)(nptr); @@ -550,8 +658,9 @@ INTERCEPTOR(long, atol, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atol); #if SANITIZER_APPLE - if (UNLIKELY(!asan_inited)) return REAL(atol)(nptr); -#endif + if (UNLIKELY(!AsanInited())) + return REAL(atol)(nptr); +# endif ENSURE_ASAN_INITED(); if (!flags()->replace_str) { return REAL(atol)(nptr); @@ -563,20 +672,6 @@ INTERCEPTOR(long, atol, const char *nptr) { return result; } -#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL -INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) { - void *ctx; - ASAN_INTERCEPTOR_ENTER(ctx, strtoll); - ENSURE_ASAN_INITED(); - if (!flags()->replace_str) { - return REAL(strtoll)(nptr, endptr, base); - } - char *real_endptr; - long long result = REAL(strtoll)(nptr, &real_endptr, base); - StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return result; -} - INTERCEPTOR(long long, atoll, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atoll); @@ -590,7 +685,6 @@ INTERCEPTOR(long long, atoll, const char *nptr) { ASAN_READ_STRING(ctx, nptr, (real_endptr - nptr) + 1); return result; } -#endif // ASAN_INTERCEPT_ATOLL_AND_STRTOLL #if ASAN_INTERCEPT___CXA_ATEXIT || ASAN_INTERCEPT_ATEXIT static void AtCxaAtexit(void *unused) { @@ -603,8 +697,9 @@ static void AtCxaAtexit(void *unused) { INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg, void *dso_handle) { #if SANITIZER_APPLE - if (UNLIKELY(!asan_inited)) return REAL(__cxa_atexit)(func, arg, dso_handle); -#endif + if (UNLIKELY(!AsanInited())) + return REAL(__cxa_atexit)(func, arg, dso_handle); +# endif ENSURE_ASAN_INITED(); #if CAN_SANITIZE_LEAKS __lsan::ScopedInterceptorDisabler disabler; @@ -656,6 +751,7 @@ void InitializeAsanInterceptors() { static bool was_called_once; CHECK(!was_called_once); was_called_once = true; + InitializePlatformInterceptors(); InitializeCommonInterceptors(); InitializeSignalInterceptors(); @@ -674,11 +770,13 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(atoi); ASAN_INTERCEPT_FUNC(atol); - ASAN_INTERCEPT_FUNC(strtol); -#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL ASAN_INTERCEPT_FUNC(atoll); + ASAN_INTERCEPT_FUNC(strtol); ASAN_INTERCEPT_FUNC(strtoll); -#endif +# if SANITIZER_GLIBC + ASAN_INTERCEPT_FUNC(__isoc23_strtol); + ASAN_INTERCEPT_FUNC(__isoc23_strtoll); +# endif // Intecept jump-related functions. ASAN_INTERCEPT_FUNC(longjmp); @@ -722,6 +820,16 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(pthread_create); #endif ASAN_INTERCEPT_FUNC(pthread_join); + ASAN_INTERCEPT_FUNC(pthread_detach); + ASAN_INTERCEPT_FUNC(pthread_exit); +# endif + +# if ASAN_INTERCEPT_TIMEDJOIN + ASAN_INTERCEPT_FUNC(pthread_timedjoin_np); +#endif + +#if ASAN_INTERCEPT_TRYJOIN + ASAN_INTERCEPT_FUNC(pthread_tryjoin_np); #endif // Intercept atexit function. @@ -741,8 +849,6 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(vfork); #endif - InitializePlatformInterceptors(); - VReport(1, "AddressSanitizer: libc interceptors initialized\n"); } diff --git a/libsanitizer/asan/asan_interceptors.h b/libsanitizer/asan/asan_interceptors.h index 9a6c22c764a..e355c1258a9 100644 --- a/libsanitizer/asan/asan_interceptors.h +++ b/libsanitizer/asan/asan_interceptors.h @@ -24,12 +24,12 @@ namespace __asan { void InitializeAsanInterceptors(); void InitializePlatformInterceptors(); -#define ENSURE_ASAN_INITED() \ - do { \ - CHECK(!asan_init_is_running); \ - if (UNLIKELY(!asan_inited)) { \ - AsanInitFromRtl(); \ - } \ +#define ENSURE_ASAN_INITED() \ + do { \ + CHECK(!AsanInitIsRunning()); \ + if (UNLIKELY(!AsanInited())) { \ + AsanInitFromRtl(); \ + } \ } while (0) } // namespace __asan @@ -42,12 +42,10 @@ void InitializePlatformInterceptors(); // Use macro to describe if specific function should be // intercepted on a given platform. #if !SANITIZER_WINDOWS -# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 1 # define ASAN_INTERCEPT__LONGJMP 1 # define ASAN_INTERCEPT_INDEX 1 # define ASAN_INTERCEPT_PTHREAD_CREATE 1 #else -# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 0 # define ASAN_INTERCEPT__LONGJMP 0 # define ASAN_INTERCEPT_INDEX 0 # define ASAN_INTERCEPT_PTHREAD_CREATE 0 @@ -78,15 +76,10 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___LONGJMP_CHK 0 #endif -#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \ - !SANITIZER_NETBSD +#if ASAN_HAS_EXCEPTIONS && !SANITIZER_SOLARIS && !SANITIZER_NETBSD && \ + (!SANITIZER_WINDOWS || (defined(__MINGW32__) && defined(__i386__))) # define ASAN_INTERCEPT___CXA_THROW 1 -# if ! defined(ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION) \ - || ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 -# else -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 0 -# endif +# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 # if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__)) # define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 1 # else @@ -117,6 +110,14 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___STRDUP 0 #endif +#if SANITIZER_GLIBC && ASAN_INTERCEPT_PTHREAD_CREATE +# define ASAN_INTERCEPT_TIMEDJOIN 1 +# define ASAN_INTERCEPT_TRYJOIN 1 +#else +# define ASAN_INTERCEPT_TIMEDJOIN 0 +# define ASAN_INTERCEPT_TRYJOIN 0 +#endif + #if SANITIZER_LINUX && \ (defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \ defined(__x86_64__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64) @@ -163,6 +164,12 @@ DECLARE_REAL(char*, strstr, const char *s1, const char *s2) # define ASAN_INTERCEPT_FUNC(name) # endif // SANITIZER_APPLE +#define ASAN_INTERCEPTOR_ENTER(ctx, func) \ + AsanInterceptorContext _ctx = {#func}; \ + ctx = (void *)&_ctx; \ + (void) ctx; +#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name) + #endif // !SANITIZER_FUCHSIA #endif // ASAN_INTERCEPTORS_H diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp index 9c316bb9574..bdf328f8920 100644 --- a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp +++ b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp @@ -11,13 +11,54 @@ // ASan versions of memcpy, memmove, and memset. //===---------------------------------------------------------------------===// +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + #include "asan_interceptors_memintrinsics.h" + +#include "asan_interceptors.h" #include "asan_report.h" #include "asan_stack.h" #include "asan_suppressions.h" using namespace __asan; +// memcpy is called during __asan_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + if (LIKELY(to != from)) { \ + CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ + } \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } else if (UNLIKELY(!AsanInited())) { \ + return internal_memcpy(to, from, size); \ + } \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_WRITE_RANGE(ctx, block, size); \ + } else if (UNLIKELY(!AsanInited())) { \ + return internal_memset(block, c, size); \ + } \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return internal_memmove(to, from, size); \ + } while (0) + void *__asan_memcpy(void *to, const void *from, uptr size) { ASAN_MEMCPY_IMPL(nullptr, to, from, size); } @@ -40,4 +81,26 @@ extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]]; extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]]; extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]]; +#else // SANITIZER_FUCHSIA + +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memmove); \ + ASAN_MEMMOVE_IMPL(ctx, to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \ + ASAN_MEMCPY_IMPL(ctx, to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memset); \ + ASAN_MEMSET_IMPL(ctx, block, c, size); \ + } while (false) + +#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc" + #endif // SANITIZER_FUCHSIA diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.h b/libsanitizer/asan/asan_interceptors_memintrinsics.h index bbc5390ceaa..eb44f8f2f72 100644 --- a/libsanitizer/asan/asan_interceptors_memintrinsics.h +++ b/libsanitizer/asan/asan_interceptors_memintrinsics.h @@ -79,43 +79,6 @@ struct AsanInterceptorContext { } \ } while (0) -// memcpy is called during __asan_init() from the internals of printf(...). -// We do not treat memcpy with to==from as a bug. -// See http://llvm.org/bugs/show_bug.cgi?id=11763. -#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - if (LIKELY(to != from)) { \ - CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ - } \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } else if (UNLIKELY(!asan_inited)) { \ - return internal_memcpy(to, from, size); \ - } \ - return REAL(memcpy)(to, from, size); \ - } while (0) - -// memset is called inside Printf. -#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - ASAN_WRITE_RANGE(ctx, block, size); \ - } else if (UNLIKELY(!asan_inited)) { \ - return internal_memset(block, c, size); \ - } \ - return REAL(memset)(block, c, size); \ - } while (0) - -#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return internal_memmove(to, from, size); \ - } while (0) - #define ASAN_READ_RANGE(ctx, offset, size) \ ACCESS_MEMORY_RANGE(ctx, offset, size, false) #define ASAN_WRITE_RANGE(ctx, offset, size) \ diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h index a5348e35b29..e2b1e9800f5 100644 --- a/libsanitizer/asan/asan_internal.h +++ b/libsanitizer/asan/asan_internal.h @@ -130,9 +130,8 @@ void InstallAtExitCheckLeaks(); if (&__asan_on_error) \ __asan_on_error() -extern int asan_inited; -// Used to avoid infinite recursion in __asan_init(). -extern bool asan_init_is_running; +bool AsanInited(); +bool AsanInitIsRunning(); // Used to avoid infinite recursion in __asan_init(). extern bool replace_intrin_cached; extern void (*death_callback)(void); // These magic values are written to shadow for better error diff --git a/libsanitizer/asan/asan_mac.cpp b/libsanitizer/asan/asan_mac.cpp index c9bd5fb8e1a..5d5146e0cde 100644 --- a/libsanitizer/asan/asan_mac.cpp +++ b/libsanitizer/asan/asan_mac.cpp @@ -130,6 +130,18 @@ typedef void* dispatch_source_t; typedef u64 dispatch_time_t; typedef void (*dispatch_function_t)(void *block); typedef void* (*worker_t)(void *block); +typedef unsigned long dispatch_mach_reason; +typedef void *dispatch_mach_msg_t; +typedef int mach_error_t; +typedef void *dispatch_mach_t; + +typedef void (*dispatch_mach_handler_function_t)(void *context, + dispatch_mach_reason reason, + dispatch_mach_msg_t message, + mach_error_t error); +typedef void (^dispatch_mach_handler_t)(dispatch_mach_reason reason, + dispatch_mach_msg_t message, + mach_error_t error); // A wrapper for the ObjC blocks used to support libdispatch. typedef struct { @@ -142,8 +154,7 @@ ALWAYS_INLINE void asan_register_worker_thread(int parent_tid, StackTrace *stack) { AsanThread *t = GetCurrentThread(); if (!t) { - t = AsanThread::Create(/* start_routine */ nullptr, /* arg */ nullptr, - parent_tid, stack, /* detached */ true); + t = AsanThread::Create(parent_tid, stack, /* detached */ true); t->Init(); asanThreadRegistry().StartThread(t->tid(), GetTid(), ThreadType::Worker, nullptr); @@ -160,7 +171,7 @@ void asan_dispatch_call_block_and_release(void *block) { VReport(2, "asan_dispatch_call_block_and_release(): " "context: %p, pthread_self: %p\n", - block, pthread_self()); + block, (void*)pthread_self()); asan_register_worker_thread(context->parent_tid, &stack); // Call the original dispatcher for the block. context->func(context->block); @@ -193,7 +204,7 @@ asan_block_context_t *alloc_asan_context(void *ctxt, dispatch_function_t func, asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); \ if (Verbosity() >= 2) { \ Report(#dispatch_x_f "(): context: %p, pthread_self: %p\n", \ - asan_ctxt, pthread_self()); \ + (void*)asan_ctxt, (void*)pthread_self()); \ PRINT_CURRENT_STACK(); \ } \ return REAL(dispatch_x_f)(dq, (void*)asan_ctxt, \ @@ -210,7 +221,7 @@ INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, GET_STACK_TRACE_THREAD; asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); if (Verbosity() >= 2) { - Report("dispatch_after_f: %p\n", asan_ctxt); + Report("dispatch_after_f: %p\n", (void*)asan_ctxt); PRINT_CURRENT_STACK(); } return REAL(dispatch_after_f)(when, dq, (void*)asan_ctxt, @@ -224,7 +235,7 @@ INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); if (Verbosity() >= 2) { Report("dispatch_group_async_f(): context: %p, pthread_self: %p\n", - asan_ctxt, pthread_self()); + (void*)asan_ctxt, (void*)pthread_self()); PRINT_CURRENT_STACK(); } REAL(dispatch_group_async_f)(group, dq, (void*)asan_ctxt, @@ -241,6 +252,8 @@ void dispatch_after(dispatch_time_t when, dispatch_queue_t queue, void dispatch_source_set_cancel_handler(dispatch_source_t ds, void(^work)(void)); void dispatch_source_set_event_handler(dispatch_source_t ds, void(^work)(void)); +dispatch_mach_t dispatch_mach_create(const char *label, dispatch_queue_t queue, + dispatch_mach_handler_t handler); } #define GET_ASAN_BLOCK(work) \ @@ -290,6 +303,34 @@ INTERCEPTOR(void, dispatch_source_set_event_handler, GET_ASAN_BLOCK(work); REAL(dispatch_source_set_event_handler)(ds, asan_block); } + +INTERCEPTOR(void *, dispatch_mach_create, const char *label, + dispatch_queue_t dq, dispatch_mach_handler_t handler) { + int parent_tid = GetCurrentTidOrInvalid(); + return REAL(dispatch_mach_create)( + label, dq, + ^(dispatch_mach_reason reason, dispatch_mach_msg_t message, + mach_error_t error) { + GET_STACK_TRACE_THREAD; + asan_register_worker_thread(parent_tid, &stack); + handler(reason, message, error); + }); +} + +INTERCEPTOR(void *, dispatch_mach_create_f, const char *label, + dispatch_queue_t dq, void *ctxt, + dispatch_mach_handler_function_t handler) { + int parent_tid = GetCurrentTidOrInvalid(); + return REAL(dispatch_mach_create)( + label, dq, + ^(dispatch_mach_reason reason, dispatch_mach_msg_t message, + mach_error_t error) { + GET_STACK_TRACE_THREAD; + asan_register_worker_thread(parent_tid, &stack); + handler(ctxt, reason, message, error); + }); +} + #endif #endif // SANITIZER_APPLE diff --git a/libsanitizer/asan/asan_malloc_linux.cpp b/libsanitizer/asan/asan_malloc_linux.cpp index bab80b96f58..0ba74c5d714 100644 --- a/libsanitizer/asan/asan_malloc_linux.cpp +++ b/libsanitizer/asan/asan_malloc_linux.cpp @@ -31,7 +31,7 @@ using namespace __asan; struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> { - static bool UseImpl() { return asan_init_is_running; } + static bool UseImpl() { return AsanInitIsRunning(); } static void OnAllocate(const void *ptr, uptr size) { # if CAN_SANITIZE_LEAKS // Suppress leaks from dlerror(). Previously dlsym hack on global array was diff --git a/libsanitizer/asan/asan_malloc_mac.cpp b/libsanitizer/asan/asan_malloc_mac.cpp index 924d1f12640..d2380ee62bf 100644 --- a/libsanitizer/asan/asan_malloc_mac.cpp +++ b/libsanitizer/asan/asan_malloc_mac.cpp @@ -23,45 +23,44 @@ using namespace __asan; #define COMMON_MALLOC_ZONE_NAME "asan" #define COMMON_MALLOC_ENTER() ENSURE_ASAN_INITED() -#define COMMON_MALLOC_SANITIZER_INITIALIZED asan_inited -#define COMMON_MALLOC_FORCE_LOCK() asan_mz_force_lock() -#define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock() -#define COMMON_MALLOC_MEMALIGN(alignment, size) \ - GET_STACK_TRACE_MALLOC; \ - void *p = asan_memalign(alignment, size, &stack, FROM_MALLOC) -#define COMMON_MALLOC_MALLOC(size) \ - GET_STACK_TRACE_MALLOC; \ - void *p = asan_malloc(size, &stack) -#define COMMON_MALLOC_REALLOC(ptr, size) \ - GET_STACK_TRACE_MALLOC; \ - void *p = asan_realloc(ptr, size, &stack); -#define COMMON_MALLOC_CALLOC(count, size) \ - GET_STACK_TRACE_MALLOC; \ - void *p = asan_calloc(count, size, &stack); -#define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ - GET_STACK_TRACE_MALLOC; \ - int res = asan_posix_memalign(memptr, alignment, size, &stack); -#define COMMON_MALLOC_VALLOC(size) \ - GET_STACK_TRACE_MALLOC; \ - void *p = asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC); -#define COMMON_MALLOC_FREE(ptr) \ - GET_STACK_TRACE_FREE; \ - asan_free(ptr, &stack, FROM_MALLOC); -#define COMMON_MALLOC_SIZE(ptr) \ - uptr size = asan_mz_size(ptr); -#define COMMON_MALLOC_FILL_STATS(zone, stats) \ - AsanMallocStats malloc_stats; \ - FillMallocStatistics(&malloc_stats); \ - CHECK(sizeof(malloc_statistics_t) == sizeof(AsanMallocStats)); \ - internal_memcpy(stats, &malloc_stats, sizeof(malloc_statistics_t)); -#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ - GET_STACK_TRACE_FREE; \ - ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); -#define COMMON_MALLOC_NAMESPACE __asan -#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 -#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 1 +# define COMMON_MALLOC_SANITIZER_INITIALIZED AsanInited() +# define COMMON_MALLOC_FORCE_LOCK() asan_mz_force_lock() +# define COMMON_MALLOC_FORCE_UNLOCK() asan_mz_force_unlock() +# define COMMON_MALLOC_MEMALIGN(alignment, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_memalign(alignment, size, &stack, FROM_MALLOC) +# define COMMON_MALLOC_MALLOC(size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_malloc(size, &stack) +# define COMMON_MALLOC_REALLOC(ptr, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_realloc(ptr, size, &stack); +# define COMMON_MALLOC_CALLOC(count, size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_calloc(count, size, &stack); +# define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ + GET_STACK_TRACE_MALLOC; \ + int res = asan_posix_memalign(memptr, alignment, size, &stack); +# define COMMON_MALLOC_VALLOC(size) \ + GET_STACK_TRACE_MALLOC; \ + void *p = asan_memalign(GetPageSizeCached(), size, &stack, FROM_MALLOC); +# define COMMON_MALLOC_FREE(ptr) \ + GET_STACK_TRACE_FREE; \ + asan_free(ptr, &stack, FROM_MALLOC); +# define COMMON_MALLOC_SIZE(ptr) uptr size = asan_mz_size(ptr); +# define COMMON_MALLOC_FILL_STATS(zone, stats) \ + AsanMallocStats malloc_stats; \ + FillMallocStatistics(&malloc_stats); \ + CHECK(sizeof(malloc_statistics_t) == sizeof(AsanMallocStats)); \ + internal_memcpy(stats, &malloc_stats, sizeof(malloc_statistics_t)); +# define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \ + GET_STACK_TRACE_FREE; \ + ReportMacMzReallocUnknown((uptr)ptr, (uptr)zone_ptr, zone_name, &stack); +# define COMMON_MALLOC_NAMESPACE __asan +# define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0 +# define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 1 -#include "sanitizer_common/sanitizer_malloc_mac.inc" +# include "sanitizer_common/sanitizer_malloc_mac.inc" namespace COMMON_MALLOC_NAMESPACE { diff --git a/libsanitizer/asan/asan_malloc_win.cpp b/libsanitizer/asan/asan_malloc_win.cpp index ff78d7646a9..7e1d04c36dd 100644 --- a/libsanitizer/asan/asan_malloc_win.cpp +++ b/libsanitizer/asan/asan_malloc_win.cpp @@ -211,7 +211,7 @@ INTERCEPTOR_WINAPI(size_t, HeapSize, HANDLE hHeap, DWORD dwFlags, // interception takes place, so if it is not owned by the RTL heap we can // pass it to the ASAN heap for inspection. if (flags()->windows_hook_rtl_allocators) { - if (!asan_inited || OWNED_BY_RTL(hHeap, lpMem)) + if (!AsanInited() || OWNED_BY_RTL(hHeap, lpMem)) return REAL(HeapSize)(hHeap, dwFlags, lpMem); } else { CHECK(dwFlags == 0 && "unsupported heap flags"); @@ -226,7 +226,7 @@ INTERCEPTOR_WINAPI(LPVOID, HeapAlloc, HANDLE hHeap, DWORD dwFlags, // If the ASAN runtime is not initialized, or we encounter an unsupported // flag, fall back to the original allocator. if (flags()->windows_hook_rtl_allocators) { - if (UNLIKELY(!asan_inited || + if (UNLIKELY(!AsanInited() || (dwFlags & HEAP_ALLOCATE_UNSUPPORTED_FLAGS) != 0)) { return REAL(HeapAlloc)(hHeap, dwFlags, dwBytes); } @@ -297,7 +297,7 @@ void *SharedReAlloc(ReAllocFunction reallocFunc, SizeFunction heapSizeFunc, // If this heap block which was allocated before the ASAN // runtime came up, use the real HeapFree function. - if (UNLIKELY(!asan_inited)) { + if (UNLIKELY(!AsanInited())) { return reallocFunc(hHeap, dwFlags, lpMem, dwBytes); } bool only_asan_supported_flags = @@ -420,7 +420,7 @@ size_t RtlSizeHeap(void* HeapHandle, DWORD Flags, void* BaseAddress); INTERCEPTOR_WINAPI(size_t, RtlSizeHeap, HANDLE HeapHandle, DWORD Flags, void* BaseAddress) { if (!flags()->windows_hook_rtl_allocators || - UNLIKELY(!asan_inited || OWNED_BY_RTL(HeapHandle, BaseAddress))) { + UNLIKELY(!AsanInited() || OWNED_BY_RTL(HeapHandle, BaseAddress))) { return REAL(RtlSizeHeap)(HeapHandle, Flags, BaseAddress); } GET_CURRENT_PC_BP_SP; @@ -448,7 +448,7 @@ INTERCEPTOR_WINAPI(void*, RtlAllocateHeap, HANDLE HeapHandle, DWORD Flags, // If the ASAN runtime is not initialized, or we encounter an unsupported // flag, fall back to the original allocator. if (!flags()->windows_hook_rtl_allocators || - UNLIKELY(!asan_inited || + UNLIKELY(!AsanInited() || (Flags & HEAP_ALLOCATE_UNSUPPORTED_FLAGS) != 0)) { return REAL(RtlAllocateHeap)(HeapHandle, Flags, Size); } diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h index 47ccf8444d3..c5f95c07a21 100644 --- a/libsanitizer/asan/asan_mapping.h +++ b/libsanitizer/asan/asan_mapping.h @@ -190,7 +190,7 @@ # elif defined(__aarch64__) # define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000 # elif defined(__powerpc64__) -# define ASAN_SHADOW_OFFSET_CONST 0x0000020000000000 +# define ASAN_SHADOW_OFFSET_CONST 0x0000100000000000 # elif defined(__s390x__) # define ASAN_SHADOW_OFFSET_CONST 0x0010000000000000 # elif SANITIZER_FREEBSD diff --git a/libsanitizer/asan/asan_poisoning.cpp b/libsanitizer/asan/asan_poisoning.cpp index 5164b7d860f..746ad61813c 100644 --- a/libsanitizer/asan/asan_poisoning.cpp +++ b/libsanitizer/asan/asan_poisoning.cpp @@ -160,10 +160,6 @@ void __asan_unpoison_memory_region(void const volatile *addr, uptr size) { return; } CHECK_LT(beg.chunk, end.chunk); - if (beg.offset > 0) { - *beg.chunk = 0; - beg.chunk++; - } REAL(memset)(beg.chunk, 0, end.chunk - beg.chunk); if (end.offset > 0 && end.value != 0) { *end.chunk = Max(end.value, end.offset); @@ -449,11 +445,14 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p, // FIXME: Two of these three checks are disabled until we fix // https://github.com/google/sanitizers/issues/258. // if (d1 != d2) - // CHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1); - if (a + granularity <= d1) - CHECK_EQ(*(u8 *)MemToShadow(a), 0); + // DCHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1); + // + // NOTE: curly brackets for the "if" below to silence a MSVC warning. + if (a + granularity <= d1) { + DCHECK_EQ(*(u8 *)MemToShadow(a), 0); + } // if (d2 + granularity <= c && c <= end) - // CHECK_EQ(*(u8 *)MemToShadow(c - granularity), + // DCHECK_EQ(*(u8 *)MemToShadow(c - granularity), // kAsanContiguousContainerOOBMagic); uptr b1 = RoundDownTo(new_end, granularity); diff --git a/libsanitizer/asan/asan_posix.cpp b/libsanitizer/asan/asan_posix.cpp index 765f4a26cd7..e1f66641617 100644 --- a/libsanitizer/asan/asan_posix.cpp +++ b/libsanitizer/asan/asan_posix.cpp @@ -138,6 +138,12 @@ void PlatformTSDDtor(void *tsd) { CHECK_EQ(0, pthread_setspecific(tsd_key, tsd)); return; } +# if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD || \ + SANITIZER_SOLARIS + // After this point it's unsafe to execute signal handlers which may be + // instrumented. It's probably not just a Linux issue. + BlockSignals(); +# endif AsanThread::TSDDtor(tsd); } #endif diff --git a/libsanitizer/asan/asan_report.cpp b/libsanitizer/asan/asan_report.cpp index f2c04342e77..7603e813115 100644 --- a/libsanitizer/asan/asan_report.cpp +++ b/libsanitizer/asan/asan_report.cpp @@ -60,9 +60,9 @@ void AppendToErrorMessageBuffer(const char *buffer) { void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte, bool in_shadow, const char *after) { Decorator d; - str->append("%s%s%x%x%s%s", before, - in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4, - byte & 15, d.Default(), after); + str->AppendF("%s%s%x%x%s%s", before, + in_shadow ? d.ShadowByte(byte) : d.MemoryByte(), byte >> 4, + byte & 15, d.Default(), after); } static void PrintZoneForPointer(uptr ptr, uptr zone_ptr, diff --git a/libsanitizer/asan/asan_report.h b/libsanitizer/asan/asan_report.h index 248e30dd42b..3540b3b4b1b 100644 --- a/libsanitizer/asan/asan_report.h +++ b/libsanitizer/asan/asan_report.h @@ -35,7 +35,8 @@ int GetGlobalsForAddress(uptr addr, __asan_global *globals, u32 *reg_sites, const char *MaybeDemangleGlobalName(const char *name); void PrintGlobalNameIfASCII(InternalScopedString *str, const __asan_global &g); -void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g); +void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g, + bool print_module_name); void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte, bool in_shadow, const char *after = "\n"); diff --git a/libsanitizer/asan/asan_rtl.cpp b/libsanitizer/asan/asan_rtl.cpp index 853083182b4..d1e7856973b 100644 --- a/libsanitizer/asan/asan_rtl.cpp +++ b/libsanitizer/asan/asan_rtl.cpp @@ -71,8 +71,17 @@ static void CheckUnwind() { } // -------------------------- Globals --------------------- {{{1 -int asan_inited; -bool asan_init_is_running; +static int asan_inited = 0; +static int asan_init_is_running = 0; + +void SetAsanInited(u32 val) { asan_inited = val; } + +void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; } + +bool AsanInited() { return asan_inited == 1; } + +bool AsanInitIsRunning() { return asan_init_is_running == 1; } + bool replace_intrin_cached; #if !ASAN_FIXED_MAPPING @@ -382,10 +391,11 @@ void PrintAddressSpaceLayout() { } static void AsanInitInternal() { - if (LIKELY(asan_inited)) return; + if (LIKELY(AsanInited())) + return; SanitizerToolName = "AddressSanitizer"; - CHECK(!asan_init_is_running && "ASan init calls itself!"); - asan_init_is_running = true; + CHECK(!AsanInitIsRunning() && "ASan init calls itself!"); + SetAsanInitIsRunning(1); CacheBinaryName(); @@ -398,7 +408,7 @@ static void AsanInitInternal() { // Stop performing init at this point if we are being loaded via // dlopen() and the platform supports it. if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) { - asan_init_is_running = false; + SetAsanInitIsRunning(0); VReport(1, "AddressSanitizer init is being performed for dlopen().\n"); return; } @@ -460,8 +470,8 @@ static void AsanInitInternal() { // On Linux AsanThread::ThreadStart() calls malloc() that's why asan_inited // should be set to 1 prior to initializing the threads. replace_intrin_cached = flags()->replace_intrin; - asan_inited = 1; - asan_init_is_running = false; + SetAsanInited(1); + SetAsanInitIsRunning(0); if (flags()->atexit) Atexit(asan_atexit); @@ -583,7 +593,7 @@ static void UnpoisonFakeStack() { using namespace __asan; void NOINLINE __asan_handle_no_return() { - if (asan_init_is_running) + if (AsanInitIsRunning()) return; if (!PlatformUnpoisonStacks()) diff --git a/libsanitizer/asan/asan_rtl_x86_64.S b/libsanitizer/asan/asan_rtl_x86_64.S index d93b5ed2a7f..0b7363018f4 100644 --- a/libsanitizer/asan/asan_rtl_x86_64.S +++ b/libsanitizer/asan/asan_rtl_x86_64.S @@ -35,35 +35,29 @@ RLABEL(reg, op, s, add): ;\ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_1(reg, op, i) \ CLABEL(reg, op, 1, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 1, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##1_asm ;\ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_2(reg, op, i) \ CLABEL(reg, op, 2, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - add $0x1,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + add $0x1,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 2, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##2_asm ;\ #define ASAN_MEMORY_ACCESS_EXTRA_CHECK_4(reg, op, i) \ CLABEL(reg, op, 4, i): ;\ - push %rcx ;\ - mov %##reg,%rcx ;\ - and $0x7,%ecx ;\ - add $0x3,%ecx ;\ - cmp %r10d,%ecx ;\ - pop %rcx ;\ + mov %##reg,%r11 ;\ + and $0x7,%r11d ;\ + add $0x3,%r11d ;\ + cmp %r10d,%r11d ;\ jl RLABEL(reg, op, 4, i);\ mov %##reg,%rdi ;\ jmp __asan_report_##op##4_asm ;\ diff --git a/libsanitizer/asan/asan_stack.cpp b/libsanitizer/asan/asan_stack.cpp index 048295d6928..764c6ac193f 100644 --- a/libsanitizer/asan/asan_stack.cpp +++ b/libsanitizer/asan/asan_stack.cpp @@ -57,7 +57,7 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { using namespace __asan; size = 0; - if (UNLIKELY(!asan_inited)) + if (UNLIKELY(!AsanInited())) return; request_fast = StackTrace::WillUseFastUnwind(request_fast); AsanThread *t = GetCurrentThread(); diff --git a/libsanitizer/asan/asan_stack.h b/libsanitizer/asan/asan_stack.h index b9575d2f427..02a76af847a 100644 --- a/libsanitizer/asan/asan_stack.h +++ b/libsanitizer/asan/asan_stack.h @@ -32,24 +32,24 @@ u32 GetMallocContextSize(); // as early as possible (in functions exposed to the user), as we generally // don't want stack trace to contain functions from ASan internals. -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - if (max_size <= 2) { \ - stack.size = max_size; \ - if (max_size > 0) { \ - stack.top_frame_bp = GET_CURRENT_FRAME(); \ - stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ - if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ - } \ - } else { \ - stack.Unwind(StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), nullptr, fast, max_size); \ +#define GET_STACK_TRACE(max_size, fast) \ + UNINITIALIZED BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) \ + stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + fast, max_size); \ } -#define GET_STACK_TRACE_FATAL(pc, bp) \ - BufferedStackTrace stack; \ - stack.Unwind(pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_FATAL(pc, bp) \ + UNINITIALIZED BufferedStackTrace stack; \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/libsanitizer/asan/asan_stats.cpp b/libsanitizer/asan/asan_stats.cpp index 9a715ea76fe..78cb30ec763 100644 --- a/libsanitizer/asan/asan_stats.cpp +++ b/libsanitizer/asan/asan_stats.cpp @@ -142,7 +142,7 @@ uptr __sanitizer_get_current_allocated_bytes() { uptr freed = stats.freed; // Return sane value if malloced < freed due to racy // way we update accumulated stats. - return (malloced > freed) ? malloced - freed : 1; + return (malloced > freed) ? malloced - freed : 0; } uptr __sanitizer_get_heap_size() { @@ -161,7 +161,7 @@ uptr __sanitizer_get_free_bytes() { + stats.malloced_redzones; // Return sane value if total_free < total_used due to racy // way we update accumulated stats. - return (total_free > total_used) ? total_free - total_used : 1; + return (total_free > total_used) ? total_free - total_used : 0; } uptr __sanitizer_get_unmapped_bytes() { diff --git a/libsanitizer/asan/asan_thread.cpp b/libsanitizer/asan/asan_thread.cpp index 3f6e58e8775..8798968947e 100644 --- a/libsanitizer/asan/asan_thread.cpp +++ b/libsanitizer/asan/asan_thread.cpp @@ -28,7 +28,7 @@ namespace __asan { // AsanThreadContext implementation. void AsanThreadContext::OnCreated(void *arg) { - CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs*>(arg); + CreateThreadContextArgs *args = static_cast<CreateThreadContextArgs *>(arg); if (args->stack) stack_id = StackDepotPut(*args->stack); thread = args->thread; @@ -40,34 +40,49 @@ void AsanThreadContext::OnFinished() { thread = nullptr; } -// MIPS requires aligned address -static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)]; static ThreadRegistry *asan_thread_registry; +static ThreadArgRetval *thread_data; static Mutex mu_for_thread_context; -static LowLevelAllocator allocator_for_thread_context; static ThreadContextBase *GetAsanThreadContext(u32 tid) { Lock lock(&mu_for_thread_context); - return new(allocator_for_thread_context) AsanThreadContext(tid); + return new (GetGlobalLowLevelAllocator()) AsanThreadContext(tid); } -ThreadRegistry &asanThreadRegistry() { +static void InitThreads() { static bool initialized; // Don't worry about thread_safety - this should be called when there is // a single thread. - if (!initialized) { - // Never reuse ASan threads: we store pointer to AsanThreadContext - // in TSD and can't reliably tell when no more TSD destructors will - // be called. It would be wrong to reuse AsanThreadContext for another - // thread before all TSD destructors will be called for it. - asan_thread_registry = - new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); - initialized = true; - } + if (LIKELY(initialized)) + return; + // Never reuse ASan threads: we store pointer to AsanThreadContext + // in TSD and can't reliably tell when no more TSD destructors will + // be called. It would be wrong to reuse AsanThreadContext for another + // thread before all TSD destructors will be called for it. + + // MIPS requires aligned address + static ALIGNED(alignof( + ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)]; + static ALIGNED(alignof( + ThreadArgRetval)) char thread_data_placeholder[sizeof(ThreadArgRetval)]; + + asan_thread_registry = + new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); + thread_data = new (thread_data_placeholder) ThreadArgRetval(); + initialized = true; +} + +ThreadRegistry &asanThreadRegistry() { + InitThreads(); return *asan_thread_registry; } +ThreadArgRetval &asanThreadArgRetval() { + InitThreads(); + return *thread_data; +} + AsanThreadContext *GetThreadContextByTidLocked(u32 tid) { return static_cast<AsanThreadContext *>( asanThreadRegistry().GetThreadLocked(tid)); @@ -75,22 +90,29 @@ AsanThreadContext *GetThreadContextByTidLocked(u32 tid) { // AsanThread implementation. -AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg, +AsanThread *AsanThread::Create(const void *start_data, uptr data_size, u32 parent_tid, StackTrace *stack, bool detached) { uptr PageSize = GetPageSizeCached(); uptr size = RoundUpTo(sizeof(AsanThread), PageSize); - AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__); - thread->start_routine_ = start_routine; - thread->arg_ = arg; + AsanThread *thread = (AsanThread *)MmapOrDie(size, __func__); + if (data_size) { + uptr availible_size = (uptr)thread + size - (uptr)(thread->start_data_); + CHECK_LE(data_size, availible_size); + internal_memcpy(thread->start_data_, start_data, data_size); + } AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; asanThreadRegistry().CreateThread(0, detached, parent_tid, &args); return thread; } +void AsanThread::GetStartData(void *out, uptr out_size) const { + internal_memcpy(out, start_data_, out_size); +} + void AsanThread::TSDDtor(void *tsd) { - AsanThreadContext *context = (AsanThreadContext*)tsd; + AsanThreadContext *context = (AsanThreadContext *)tsd; VReport(1, "T%d TSDDtor\n", context->tid); if (context->thread) context->thread->Destroy(); @@ -144,8 +166,7 @@ void AsanThread::StartSwitchFiber(FakeStack **fake_stack_save, uptr bottom, current_fake_stack->Destroy(this->tid()); } -void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, - uptr *bottom_old, +void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old, uptr *size_old) { if (!atomic_load(&stack_switching_, memory_order_relaxed)) { Report("ERROR: finishing a fiber switch that has not started\n"); @@ -171,7 +192,8 @@ void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, inline AsanThread::StackBounds AsanThread::GetStackBounds() const { if (!atomic_load(&stack_switching_, memory_order_acquire)) { // Make sure the stack bounds are fully initialized. - if (stack_bottom_ >= stack_top_) return {0, 0}; + if (stack_bottom_ >= stack_top_) + return {0, 0}; return {stack_bottom_, stack_top_}; } char local; @@ -184,13 +206,9 @@ inline AsanThread::StackBounds AsanThread::GetStackBounds() const { return {stack_bottom_, stack_top_}; } -uptr AsanThread::stack_top() { - return GetStackBounds().top; -} +uptr AsanThread::stack_top() { return GetStackBounds().top; } -uptr AsanThread::stack_bottom() { - return GetStackBounds().bottom; -} +uptr AsanThread::stack_bottom() { return GetStackBounds().bottom; } uptr AsanThread::stack_size() { const auto bounds = GetStackBounds(); @@ -211,8 +229,8 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { // This CAS checks if the state was 0 and if so changes it to state 1, // if that was successful, it initializes the pointer. if (atomic_compare_exchange_strong( - reinterpret_cast<atomic_uintptr_t *>(&fake_stack_), &old_val, 1UL, - memory_order_relaxed)) { + reinterpret_cast<atomic_uintptr_t *>(&fake_stack_), &old_val, 1UL, + memory_order_relaxed)) { uptr stack_size_log = Log2(RoundUpToPowerOfTwo(stack_size)); CHECK_LE(flags()->min_uar_stack_size_log, flags()->max_uar_stack_size_log); stack_size_log = @@ -261,36 +279,17 @@ void AsanThread::Init(const InitOptions *options) { // asan_fuchsia.c definies CreateMainThread and SetThreadStackAndTls. #if !SANITIZER_FUCHSIA -thread_return_t AsanThread::ThreadStart(tid_t os_id) { +void AsanThread::ThreadStart(tid_t os_id) { Init(); asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr); - if (common_flags()->use_sigaltstack) SetAlternateSignalStack(); - - if (!start_routine_) { - // start_routine_ == 0 if we're on the main thread or on one of the - // OS X libdispatch worker threads. But nobody is supposed to call - // ThreadStart() for the worker threads. - CHECK_EQ(tid(), 0); - return 0; - } - - thread_return_t res = start_routine_(arg_); - - // On POSIX systems we defer this to the TSD destructor. LSan will consider - // the thread's memory as non-live from the moment we call Destroy(), even - // though that memory might contain pointers to heap objects which will be - // cleaned up by a user-defined TSD destructor. Thus, calling Destroy() before - // the TSD destructors have run might cause false positives in LSan. - if (!SANITIZER_POSIX) - this->Destroy(); - - return res; + if (common_flags()->use_sigaltstack) + SetAlternateSignalStack(); } AsanThread *CreateMainThread() { AsanThread *main_thread = AsanThread::Create( - /* start_routine */ nullptr, /* arg */ nullptr, /* parent_tid */ kMainTid, + /* parent_tid */ kMainTid, /* stack */ nullptr, /* detached */ true); SetCurrentThread(main_thread); main_thread->ThreadStart(internal_getpid()); @@ -341,14 +340,14 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, bottom = fake_stack->AddrIsInFakeStack(addr); CHECK(bottom); access->offset = addr - bottom; - access->frame_pc = ((uptr*)bottom)[2]; - access->frame_descr = (const char *)((uptr*)bottom)[1]; + access->frame_pc = ((uptr *)bottom)[2]; + access->frame_descr = (const char *)((uptr *)bottom)[1]; return true; } uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY); - u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); - u8 *shadow_bottom = (u8*)MemToShadow(bottom); + u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8 *)MemToShadow(bottom); while (shadow_ptr >= shadow_bottom && *shadow_ptr != kAsanStackLeftRedzoneMagic) { @@ -370,7 +369,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, CHECK(ptr[0] == kCurrentStackFrameMagic); access->offset = addr - (uptr)ptr; access->frame_pc = ptr[2]; - access->frame_descr = (const char*)ptr[1]; + access->frame_descr = (const char *)ptr[1]; return true; } @@ -388,8 +387,8 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) { } uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. - u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); - u8 *shadow_bottom = (u8*)MemToShadow(bottom); + u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8 *)MemToShadow(bottom); while (shadow_ptr >= shadow_bottom && (*shadow_ptr != kAsanStackLeftRedzoneMagic && @@ -473,16 +472,23 @@ void EnsureMainThreadIDIsCorrect() { __asan::AsanThread *GetAsanThreadByOsIDLocked(tid_t os_id) { __asan::AsanThreadContext *context = static_cast<__asan::AsanThreadContext *>( __asan::asanThreadRegistry().FindThreadContextByOsIDLocked(os_id)); - if (!context) return nullptr; + if (!context) + return nullptr; return context->thread; } -} // namespace __asan +} // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 namespace __lsan { -void LockThreadRegistry() { __asan::asanThreadRegistry().Lock(); } +void LockThreads() { + __asan::asanThreadRegistry().Lock(); + __asan::asanThreadArgRetval().Lock(); +} -void UnlockThreadRegistry() { __asan::asanThreadRegistry().Unlock(); } +void UnlockThreads() { + __asan::asanThreadArgRetval().Unlock(); + __asan::asanThreadRegistry().Unlock(); +} static ThreadRegistry *GetAsanThreadRegistryLocked() { __asan::asanThreadRegistry().CheckLocked(); @@ -495,7 +501,8 @@ bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end, uptr *tls_begin, uptr *tls_end, uptr *cache_begin, uptr *cache_end, DTLS **dtls) { __asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id); - if (!t) return false; + if (!t) + return false; *stack_begin = t->stack_bottom(); *stack_end = t->stack_top(); *tls_begin = t->tls_begin(); @@ -536,33 +543,7 @@ void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) { } void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) { - GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked( - [](ThreadContextBase *tctx, void *ptrs) { - // Look for the arg pointer of threads that have been created or are - // running. This is necessary to prevent false positive leaks due to the - // AsanThread holding the only live reference to a heap object. This - // can happen because the `pthread_create()` interceptor doesn't wait - // for the child thread to start before returning and thus loosing the - // the only live reference to the heap object on the stack. - - __asan::AsanThreadContext *atctx = - static_cast<__asan::AsanThreadContext *>(tctx); - - // Note ThreadStatusRunning is required because there is a small window - // where the thread status switches to `ThreadStatusRunning` but the - // `arg` pointer still isn't on the stack yet. - if (atctx->status != ThreadStatusCreated && - atctx->status != ThreadStatusRunning) - return; - - uptr thread_arg = reinterpret_cast<uptr>(atctx->thread->get_arg()); - if (!thread_arg) - return; - - auto ptrsVec = reinterpret_cast<InternalMmapVector<uptr> *>(ptrs); - ptrsVec->push_back(thread_arg); - }, - ptrs); + __asan::asanThreadArgRetval().GetAllPtrsLocked(ptrs); } void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) { @@ -575,11 +556,7 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) { threads); } -void FinishThreadLocked(u32 tid) { - GetAsanThreadRegistryLocked()->FinishThread(tid); -} - -} // namespace __lsan +} // namespace __lsan // ---------------------- Interface ---------------- {{{1 using namespace __asan; @@ -593,20 +570,18 @@ void __sanitizer_start_switch_fiber(void **fakestacksave, const void *bottom, VReport(1, "__asan_start_switch_fiber called from unknown thread\n"); return; } - t->StartSwitchFiber((FakeStack**)fakestacksave, (uptr)bottom, size); + t->StartSwitchFiber((FakeStack **)fakestacksave, (uptr)bottom, size); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_finish_switch_fiber(void* fakestack, - const void **bottom_old, +void __sanitizer_finish_switch_fiber(void *fakestack, const void **bottom_old, uptr *size_old) { AsanThread *t = GetCurrentThread(); if (!t) { VReport(1, "__asan_finish_switch_fiber called from unknown thread\n"); return; } - t->FinishSwitchFiber((FakeStack*)fakestack, - (uptr*)bottom_old, - (uptr*)size_old); + t->FinishSwitchFiber((FakeStack *)fakestack, (uptr *)bottom_old, + (uptr *)size_old); } } diff --git a/libsanitizer/asan/asan_thread.h b/libsanitizer/asan/asan_thread.h index 801a3960ec6..62f1b5337fe 100644 --- a/libsanitizer/asan/asan_thread.h +++ b/libsanitizer/asan/asan_thread.h @@ -15,11 +15,12 @@ #define ASAN_THREAD_H #include "asan_allocator.h" -#include "asan_internal.h" #include "asan_fake_stack.h" +#include "asan_internal.h" #include "asan_stats.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_thread_arg_retval.h" #include "sanitizer_common/sanitizer_thread_registry.h" namespace __sanitizer { @@ -55,18 +56,32 @@ class AsanThreadContext final : public ThreadContextBase { // AsanThreadContext objects are never freed, so we need many of them. COMPILER_CHECK(sizeof(AsanThreadContext) <= 256); +#if defined(_MSC_VER) && !defined(__clang__) +// MSVC raises a warning about a nonstandard extension being used for the 0 +// sized element in this array. Disable this for warn-as-error builds. +# pragma warning(push) +# pragma warning(disable : 4200) +#endif + // AsanThread are stored in TSD and destroyed when the thread dies. class AsanThread { public: - static AsanThread *Create(thread_callback_t start_routine, void *arg, - u32 parent_tid, StackTrace *stack, bool detached); + template <typename T> + static AsanThread *Create(const T &data, u32 parent_tid, StackTrace *stack, + bool detached) { + return Create(&data, sizeof(data), parent_tid, stack, detached); + } + static AsanThread *Create(u32 parent_tid, StackTrace *stack, bool detached) { + return Create(nullptr, 0, parent_tid, stack, detached); + } static void TSDDtor(void *tsd); void Destroy(); struct InitOptions; void Init(const InitOptions *options = nullptr); - thread_return_t ThreadStart(tid_t os_id); + void ThreadStart(tid_t os_id); + thread_return_t RunThread(); uptr stack_top(); uptr stack_bottom(); @@ -129,12 +144,18 @@ class AsanThread { void *extra_spill_area() { return &extra_spill_area_; } - void *get_arg() { return arg_; } + template <typename T> + void GetStartData(T &data) const { + GetStartData(&data, sizeof(data)); + } private: // NOTE: There is no AsanThread constructor. It is allocated // via mmap() and *must* be valid in zero-initialized state. + static AsanThread *Create(const void *start_data, uptr data_size, + u32 parent_tid, StackTrace *stack, bool detached); + void SetThreadStackAndTls(const InitOptions *options); void ClearShadowForThreadStackAndTLS(); @@ -146,9 +167,9 @@ class AsanThread { }; StackBounds GetStackBounds() const; + void GetStartData(void *out, uptr out_size) const; + AsanThreadContext *context_; - thread_callback_t start_routine_; - void *arg_; uptr stack_top_; uptr stack_bottom_; @@ -167,10 +188,17 @@ class AsanThread { AsanStats stats_; bool unwinding_; uptr extra_spill_area_; + + char start_data_[]; }; +#if defined(_MSC_VER) && !defined(__clang__) +# pragma warning(pop) +#endif + // Returns a single instance of registry. ThreadRegistry &asanThreadRegistry(); +ThreadArgRetval &asanThreadArgRetval(); // Must be called under ThreadRegistryLock. AsanThreadContext *GetThreadContextByTidLocked(u32 tid); diff --git a/libsanitizer/asan/asan_win.cpp b/libsanitizer/asan/asan_win.cpp index 7dbd7ab98a1..d5a30f471e2 100644 --- a/libsanitizer/asan/asan_win.cpp +++ b/libsanitizer/asan/asan_win.cpp @@ -131,10 +131,22 @@ INTERCEPTOR(int, _except_handler4, void *a, void *b, void *c, void *d) { } #endif +struct ThreadStartParams { + thread_callback_t start_routine; + void *arg; +}; + static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { AsanThread *t = (AsanThread *)arg; SetCurrentThread(t); - return t->ThreadStart(GetTid()); + t->ThreadStart(GetTid()); + + ThreadStartParams params; + t->GetStartData(params); + + auto res = (*params.start_routine)(params.arg); + t->Destroy(); // POSIX calls this from TSD destructor. + return res; } INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, @@ -148,8 +160,8 @@ INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, // one. This is a bandaid fix for PR22025. bool detached = false; // FIXME: how can we determine it on Windows? u32 current_tid = GetCurrentTidOrInvalid(); - AsanThread *t = - AsanThread::Create(start_routine, arg, current_tid, &stack, detached); + ThreadStartParams params = {start_routine, arg}; + AsanThread *t = AsanThread::Create(params, current_tid, &stack, detached); return REAL(CreateThread)(security, stack_size, asan_thread_start, t, thr_flags, tid); } @@ -159,6 +171,8 @@ INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, namespace __asan { void InitializePlatformInterceptors() { + __interception::SetErrorReportCallback(Report); + // The interceptors were not designed to be removable, so we have to keep this // module alive for the life of the process. HMODULE pinned; @@ -194,9 +208,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { } void FlushUnneededASanShadowMemory(uptr p, uptr size) { + // Only asan on 64-bit Windows supports committing shadow memory on demand. +#if SANITIZER_WINDOWS64 // Since asan's mapping is compacting, the shadow chunk may be // not page-aligned, so we only flush the page-aligned portion. ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); +#endif } // ---------------------- TSD ---------------- {{{ diff --git a/libsanitizer/asan/asan_win_dll_thunk.cpp b/libsanitizer/asan/asan_win_dll_thunk.cpp index e3a90f18ed8..0fa636bec0d 100644 --- a/libsanitizer/asan/asan_win_dll_thunk.cpp +++ b/libsanitizer/asan/asan_win_dll_thunk.cpp @@ -65,6 +65,7 @@ INTERCEPT_WRAP_W_W(_expand_dbg) INTERCEPT_LIBRARY_FUNCTION(atoi); INTERCEPT_LIBRARY_FUNCTION(atol); +INTERCEPT_LIBRARY_FUNCTION(atoll); INTERCEPT_LIBRARY_FUNCTION(frexp); INTERCEPT_LIBRARY_FUNCTION(longjmp); #if SANITIZER_INTERCEPT_MEMCHR @@ -91,6 +92,7 @@ INTERCEPT_LIBRARY_FUNCTION(strspn); INTERCEPT_LIBRARY_FUNCTION(strstr); INTERCEPT_LIBRARY_FUNCTION(strtok); INTERCEPT_LIBRARY_FUNCTION(strtol); +INTERCEPT_LIBRARY_FUNCTION(strtoll); INTERCEPT_LIBRARY_FUNCTION(wcslen); INTERCEPT_LIBRARY_FUNCTION(wcsnlen); diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp index 26aae9b4869..2f6cb10caf1 100644 --- a/libsanitizer/hwasan/hwasan.cpp +++ b/libsanitizer/hwasan/hwasan.cpp @@ -86,9 +86,11 @@ static void InitializeFlags() { cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); // Sigtrap is used in error reporting. cf.handle_sigtrap = kHandleSignalExclusive; - // For now only tested on Linux. Other plantforms can be turned on as they - // become ready. - cf.detect_leaks = cf.detect_leaks && SANITIZER_LINUX && !SANITIZER_ANDROID; + // For now only tested on Linux and Fuchsia. Other plantforms can be turned + // on as they become ready. + constexpr bool can_detect_leaks = + (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA; + cf.detect_leaks = cf.detect_leaks && can_detect_leaks; #if SANITIZER_ANDROID // Let platform handle other signals. It is better at reporting them then we @@ -170,7 +172,7 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) { auto sds = StackDepotGetStats(); AllocatorStatCounters asc; GetAllocatorStats(asc); - s.append( + s.AppendF( "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" " heap: %zd", @@ -290,14 +292,20 @@ static bool InitializeSingleGlobal(const hwasan_global &global) { } static void InitLoadedGlobals() { - dl_iterate_phdr( - [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int { - for (const hwasan_global &global : HwasanGlobalsFor( - info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum)) - InitializeSingleGlobal(global); - return 0; - }, - nullptr); + // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on + // the startup path which calls into __hwasan_library_loaded on all + // initially loaded modules, so explicitly registering the globals here + // isn't needed. + if constexpr (!SANITIZER_FUCHSIA) { + dl_iterate_phdr( + [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int { + for (const hwasan_global &global : HwasanGlobalsFor( + info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum)) + InitializeSingleGlobal(global); + return 0; + }, + nullptr); + } } // Prepare to run instrumented code on the main thread. @@ -364,13 +372,7 @@ __attribute__((constructor(0))) void __hwasan_init() { DisableCoreDumperIfNecessary(); InitInstrumentation(); - if constexpr (!SANITIZER_FUCHSIA) { - // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on - // the startup path which calls into __hwasan_library_loaded on all - // initially loaded modules, so explicitly registering the globals here - // isn't needed. - InitLoadedGlobals(); - } + InitLoadedGlobals(); // Needs to be called here because flags()->random_tags might not have been // initialized when InitInstrumentation() was called. @@ -530,6 +532,56 @@ void __hwasan_load16_noabort(uptr p) { CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); } +void __hwasan_loadN_match_all(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); +} +void __hwasan_load1_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p); +} +void __hwasan_load2_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p); +} +void __hwasan_load4_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p); +} +void __hwasan_load8_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p); +} +void __hwasan_load16_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p); +} + +void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz); +} +void __hwasan_load1_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p); +} +void __hwasan_load2_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p); +} +void __hwasan_load4_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p); +} +void __hwasan_load8_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p); +} +void __hwasan_load16_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); +} + void __hwasan_storeN(uptr p, uptr sz) { CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); } @@ -568,6 +620,56 @@ void __hwasan_store16_noabort(uptr p) { CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); } +void __hwasan_storeN_match_all(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); +} +void __hwasan_store1_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p); +} +void __hwasan_store2_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p); +} +void __hwasan_store4_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p); +} +void __hwasan_store8_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p); +} +void __hwasan_store16_match_all(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p); +} + +void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz); +} +void __hwasan_store1_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p); +} +void __hwasan_store2_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p); +} +void __hwasan_store4_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p); +} +void __hwasan_store8_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p); +} +void __hwasan_store16_match_all_noabort(uptr p, u8 match_all_tag) { + if (GetTagFromPointer(p) != match_all_tag) + CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); +} + void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { TagMemoryAligned(UntagAddr(p), sz, tag); } @@ -579,7 +681,7 @@ uptr __hwasan_tag_pointer(uptr p, u8 tag) { void __hwasan_handle_longjmp(const void *sp_dst) { uptr dst = (uptr)sp_dst; // HWASan does not support tagged SP. - CHECK(GetTagFromPointer(dst) == 0); + CHECK_EQ(GetTagFromPointer(dst), 0); uptr sp = (uptr)__builtin_frame_address(0); static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M diff --git a/libsanitizer/hwasan/hwasan_allocation_functions.cpp b/libsanitizer/hwasan/hwasan_allocation_functions.cpp index 59ad633879b..75d91ed09ce 100644 --- a/libsanitizer/hwasan/hwasan_allocation_functions.cpp +++ b/libsanitizer/hwasan/hwasan_allocation_functions.cpp @@ -159,13 +159,13 @@ void *__sanitizer_malloc(uptr size) { // Fuchsia does not use WRAP/wrappers used for the interceptor infrastructure. # define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ - ARGS) ALIAS("__sanitizer_" #FN) + ARGS) ALIAS(__sanitizer_##FN) #else # define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ - ALIAS("__sanitizer_" #FN); \ + ALIAS(__sanitizer_##FN); \ extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ - ARGS) ALIAS("__sanitizer_" #FN) + ARGS) ALIAS(__sanitizer_##FN) #endif INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, diff --git a/libsanitizer/hwasan/hwasan_allocator.cpp b/libsanitizer/hwasan/hwasan_allocator.cpp index 3b59741df6e..d21ba024a20 100644 --- a/libsanitizer/hwasan/hwasan_allocator.cpp +++ b/libsanitizer/hwasan/hwasan_allocator.cpp @@ -149,8 +149,9 @@ void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); - allocator.Init(common_flags()->allocator_release_to_os_interval_ms, - GetAliasRegionStart()); + allocator.InitLinkerInitialized( + common_flags()->allocator_release_to_os_interval_ms, + GetAliasRegionStart()); for (uptr i = 0; i < sizeof(tail_magic); i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); if (common_flags()->max_allocation_size_mb) { @@ -165,8 +166,11 @@ void HwasanAllocatorLock() { allocator.ForceLock(); } void HwasanAllocatorUnlock() { allocator.ForceUnlock(); } -void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) { +void AllocatorThreadStart(AllocatorCache *cache) { allocator.InitCache(cache); } + +void AllocatorThreadFinish(AllocatorCache *cache) { allocator.SwallowCache(cache); + allocator.DestroyCache(cache); } static uptr TaggedSize(uptr size) { @@ -230,28 +234,23 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, } void *user_ptr = allocated; - // Tagging can only be skipped when both tag_in_malloc and tag_in_free are - // false. When tag_in_malloc = false and tag_in_free = true malloc needs to - // retag to 0. if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) && - (flags()->tag_in_malloc || flags()->tag_in_free) && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { - if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) { - tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag; - uptr tag_size = orig_size ? orig_size : 1; - uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment); - user_ptr = - (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag); - if (full_granule_size != tag_size) { - u8 *short_granule = - reinterpret_cast<u8 *>(allocated) + full_granule_size; - TagMemoryAligned((uptr)short_granule, kShadowAlignment, - tag_size % kShadowAlignment); - short_granule[kShadowAlignment - 1] = tag; - } - } else { - user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0); + atomic_load_relaxed(&hwasan_allocator_tagging_enabled) && + flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) { + tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag; + uptr tag_size = orig_size ? orig_size : 1; + uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment); + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag); + if (full_granule_size != tag_size) { + u8 *short_granule = reinterpret_cast<u8 *>(allocated) + full_granule_size; + TagMemoryAligned((uptr)short_granule, kShadowAlignment, + tag_size % kShadowAlignment); + short_granule[kShadowAlignment - 1] = tag; } + } else { + // Tagging can not be completely skipped. If it's disabled, we need to tag + // with zeros. + user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0); } Metadata *meta = @@ -261,7 +260,7 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment, : __lsan::kDirectlyLeaked); #endif meta->SetAllocated(StackDepotPut(*stack), orig_size); - RunMallocHooks(user_ptr, size); + RunMallocHooks(user_ptr, orig_size); return user_ptr; } @@ -288,8 +287,6 @@ static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr, static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { CHECK(tagged_ptr); - RunFreeHooks(tagged_ptr); - void *untagged_ptr = UntagPtr(tagged_ptr); if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr)) @@ -304,6 +301,9 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr)); return; } + + RunFreeHooks(tagged_ptr); + uptr orig_size = meta->GetRequestedSize(); u32 free_context_id = StackDepotPut(*stack); u32 alloc_context_id = meta->GetAllocStackId(); @@ -340,7 +340,8 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size); } if (in_taggable_region && flags()->tag_in_free && malloc_bisect(stack, 0) && - atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { + atomic_load_relaxed(&hwasan_allocator_tagging_enabled) && + allocator.FromPrimary(untagged_ptr) /* Secondary 0-tag and unmap.*/) { // Always store full 8-bit tags on free to maximize UAF detection. tag_t tag; if (t) { @@ -437,6 +438,15 @@ static uptr AllocationSize(const void *p) { return b->GetRequestedSize(); } +static uptr AllocationSizeFast(const void *p) { + const void *untagged_ptr = UntagPtr(p); + void *aligned_ptr = reinterpret_cast<void *>( + RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)); + Metadata *meta = + reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr)); + return meta->GetRequestedSize(); +} + void *hwasan_malloc(uptr size, StackTrace *stack) { return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false)); } @@ -675,4 +685,11 @@ const void *__sanitizer_get_allocated_begin(const void *p) { uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); } +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = AllocationSizeFast(p); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); } diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h index ecf3f6816fc..2ada2a0b185 100644 --- a/libsanitizer/hwasan/hwasan_allocator.h +++ b/libsanitizer/hwasan/hwasan_allocator.h @@ -54,6 +54,10 @@ static_assert(sizeof(Metadata) == 16); struct HwasanMapUnmapCallback { void OnMap(uptr p, uptr size) const { UpdateMemoryUsage(); } + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const { + UpdateMemoryUsage(); + } void OnUnmap(uptr p, uptr size) const { // We are about to unmap a chunk of user memory. // It can return as user-requested mmap() or another thread stack. @@ -88,7 +92,8 @@ typedef SizeClassAllocator64<AP64> PrimaryAllocator; typedef CombinedAllocator<PrimaryAllocator> Allocator; typedef Allocator::AllocatorCache AllocatorCache; -void AllocatorSwallowThreadLocalCache(AllocatorCache *cache); +void AllocatorThreadStart(AllocatorCache *cache); +void AllocatorThreadFinish(AllocatorCache *cache); class HwasanChunkView { public: diff --git a/libsanitizer/hwasan/hwasan_exceptions.cpp b/libsanitizer/hwasan/hwasan_exceptions.cpp index c9968a5e360..bf700bf5683 100644 --- a/libsanitizer/hwasan/hwasan_exceptions.cpp +++ b/libsanitizer/hwasan/hwasan_exceptions.cpp @@ -62,7 +62,8 @@ __hwasan_personality_wrapper(int version, _Unwind_Action actions, #error Unsupported architecture #endif uptr sp = get_cfa(context); - TagMemory(sp, fp - sp, 0); + TagMemory(UntagAddr(sp), UntagAddr(fp) - UntagAddr(sp), + GetTagFromPointer(sp)); } return rc; diff --git a/libsanitizer/hwasan/hwasan_globals.cpp b/libsanitizer/hwasan/hwasan_globals.cpp index d71bcd792e1..7e0f3df20dd 100644 --- a/libsanitizer/hwasan/hwasan_globals.cpp +++ b/libsanitizer/hwasan/hwasan_globals.cpp @@ -13,6 +13,8 @@ #include "hwasan_globals.h" +#include "sanitizer_common/sanitizer_array_ref.h" + namespace __hwasan { enum { NT_LLVM_HWASAN_GLOBALS = 3 }; diff --git a/libsanitizer/hwasan/hwasan_globals.h b/libsanitizer/hwasan/hwasan_globals.h index fd7adf7a058..94cd53e1888 100644 --- a/libsanitizer/hwasan/hwasan_globals.h +++ b/libsanitizer/hwasan/hwasan_globals.h @@ -16,6 +16,7 @@ #include <link.h> +#include "sanitizer_common/sanitizer_array_ref.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_internal_defs.h" diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp index 67edba40b5b..d9237cf9b8e 100644 --- a/libsanitizer/hwasan/hwasan_interceptors.cpp +++ b/libsanitizer/hwasan/hwasan_interceptors.cpp @@ -14,10 +14,17 @@ // sanitizer_common/sanitizer_common_interceptors.h //===----------------------------------------------------------------------===// +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + #include "hwasan.h" +#include "hwasan_allocator.h" #include "hwasan_checks.h" +#include "hwasan_mapping.h" +#include "hwasan_platform_interceptors.h" #include "hwasan_thread.h" +#include "hwasan_thread_list.h" #include "interception/interception.h" +#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -25,21 +32,47 @@ using namespace __hwasan; -#if HWASAN_WITH_INTERCEPTORS - -struct ThreadStartArg { - thread_callback_t callback; - void *param; - __sanitizer_sigset_t starting_sigset_; +struct HWAsanInterceptorContext { + const char *interceptor_name; }; -static void *HwasanThreadStartFunc(void *arg) { - __hwasan_thread_enter(); - ThreadStartArg A = *reinterpret_cast<ThreadStartArg*>(arg); - SetSigProcMask(&A.starting_sigset_, nullptr); - UnmapOrDie(arg, GetPageSizeCached()); - return A.callback(A.param); -} +# define ACCESS_MEMORY_RANGE(ctx, offset, size, access) \ + do { \ + __hwasan::CheckAddressSized<ErrorAction::Abort, access>((uptr)offset, \ + size); \ + } while (0) + +# define HWASAN_READ_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, AccessType::Load) +# define HWASAN_WRITE_RANGE(ctx, offset, size) \ + ACCESS_MEMORY_RANGE(ctx, offset, size, AccessType::Store) + +# if !SANITIZER_APPLE +# define HWASAN_INTERCEPT_FUNC(name) \ + do { \ + if (!INTERCEPT_FUNCTION(name)) \ + VReport(1, "HWAddressSanitizer: failed to intercept '%s'\n", #name); \ + } while (0) +# define HWASAN_INTERCEPT_FUNC_VER(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver)) \ + VReport(1, "HWAddressSanitizer: failed to intercept '%s@@%s'\n", \ + #name, ver); \ + } while (0) +# define HWASAN_INTERCEPT_FUNC_VER_UNVERSIONED_FALLBACK(name, ver) \ + do { \ + if (!INTERCEPT_FUNCTION_VER(name, ver) && !INTERCEPT_FUNCTION(name)) \ + VReport( \ + 1, "HWAddressSanitizer: failed to intercept '%s@@%s' or '%s'\n", \ + #name, ver, #name); \ + } while (0) + +# else +// OS X interceptors don't need to be initialized with INTERCEPT_FUNCTION. +# define HWASAN_INTERCEPT_FUNC(name) +# endif // SANITIZER_APPLE + +# if HWASAN_WITH_INTERCEPTORS # define COMMON_SYSCALL_PRE_READ_RANGE(p, s) __hwasan_loadN((uptr)p, (uptr)s) # define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ @@ -57,26 +90,251 @@ static void *HwasanThreadStartFunc(void *arg) { # include "sanitizer_common/sanitizer_common_syscalls.inc" # include "sanitizer_common/sanitizer_syscalls_netbsd.inc" -INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), - void * param) { +# define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + HWASAN_WRITE_RANGE(ctx, ptr, size) + +# define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + HWASAN_READ_RANGE(ctx, ptr, size) + +# define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + HWAsanInterceptorContext _ctx = {#func}; \ + ctx = (void *)&_ctx; \ + do { \ + (void)(ctx); \ + (void)(func); \ + } while (false) + +# define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ + do { \ + (void)(ctx); \ + (void)(path); \ + } while (false) + +# define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ + do { \ + (void)(ctx); \ + (void)(fd); \ + } while (false) + +# define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ + do { \ + (void)(ctx); \ + (void)(fd); \ + } while (false) + +# define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ + do { \ + (void)(ctx); \ + (void)(fd); \ + (void)(newfd); \ + } while (false) + +# define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ + do { \ + (void)(ctx); \ + (void)(name); \ + } while (false) + +# define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ + do { \ + (void)(ctx); \ + (void)(thread); \ + (void)(name); \ + } while (false) + +# define COMMON_INTERCEPTOR_BLOCK_REAL(name) \ + do { \ + (void)(name); \ + } while (false) + +# define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ + return internal_memset(dst, v, size); \ + COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ + if (MemIsApp(UntagAddr(reinterpret_cast<uptr>(dst))) && \ + common_flags()->intercept_intrin) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + return REAL(memset)(dst, v, size); \ + } + +# define COMMON_INTERCEPTOR_STRERROR() \ + do { \ + } while (false) + +# define COMMON_INTERCEPT_FUNCTION(name) HWASAN_INTERCEPT_FUNC(name) + +# define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED (!hwasan_inited) + +// The main purpose of the mmap interceptor is to prevent the user from +// allocating on top of shadow pages. +// +// For compatibility, it does not tag pointers, nor does it allow +// MAP_FIXED in combination with a tagged pointer. (Since mmap itself +// will not return a tagged pointer, the tagged pointer must have come +// from elsewhere, such as the secondary allocator, which makes it a +// very odd usecase.) +template <class Mmap> +static void *mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length, + int prot, int flags, int fd, OFF64_T offset) { + if (addr) { + if (flags & map_fixed) CHECK_EQ(addr, UntagPtr(addr)); + + addr = UntagPtr(addr); + } + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + void *end_addr = (char *)addr + (rounded_length - 1); + if (addr && length && + (!MemIsApp(reinterpret_cast<uptr>(addr)) || + !MemIsApp(reinterpret_cast<uptr>(end_addr)))) { + // User requested an address that is incompatible with HWASan's + // memory layout. Use a different address if allowed, else fail. + if (flags & map_fixed) { + errno = errno_EINVAL; + return (void *)-1; + } else { + addr = nullptr; + } + } + void *res = real_mmap(addr, length, prot, flags, fd, offset); + if (length && res != (void *)-1) { + uptr beg = reinterpret_cast<uptr>(res); + DCHECK(IsAligned(beg, GetPageSize())); + if (!MemIsApp(beg) || !MemIsApp(beg + rounded_length - 1)) { + // Application has attempted to map more memory than is supported by + // HWASan. Act as if we ran out of memory. + internal_munmap(res, length); + errno = errno_ENOMEM; + return (void *)-1; + } + __hwasan::TagMemoryAligned(beg, rounded_length, 0); + } + + return res; +} + +template <class Munmap> +static int munmap_interceptor(Munmap real_munmap, void *addr, SIZE_T length) { + // We should not tag if munmap fail, but it's to late to tag after + // real_munmap, as the pages could be mmaped by another thread. + uptr beg = reinterpret_cast<uptr>(addr); + if (length && IsAligned(beg, GetPageSize())) { + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + // Protect from unmapping the shadow. + if (!MemIsApp(beg) || !MemIsApp(beg + rounded_length - 1)) { + errno = errno_EINVAL; + return -1; + } + __hwasan::TagMemoryAligned(beg, rounded_length, 0); + } + return real_munmap(addr, length); +} + +# define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, \ + fd, offset) \ + do { \ + (void)(ctx); \ + return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off); \ + } while (false) + +# define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, length) \ + do { \ + (void)(ctx); \ + return munmap_interceptor(REAL(munmap), addr, sz); \ + } while (false) + +# include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc" +# include "sanitizer_common/sanitizer_common_interceptors.inc" + +struct ThreadStartArg { + __sanitizer_sigset_t starting_sigset_; +}; + +static void *HwasanThreadStartFunc(void *arg) { + __hwasan_thread_enter(); + SetSigProcMask(&reinterpret_cast<ThreadStartArg *>(arg)->starting_sigset_, + nullptr); + InternalFree(arg); + auto self = GetThreadSelf(); + auto args = hwasanThreadArgRetval().GetArgs(self); + void *retval = (*args.routine)(args.arg_retval); + hwasanThreadArgRetval().Finish(self, retval); + return retval; +} + +extern "C" { +int pthread_attr_getdetachstate(void *attr, int *v); +} + +INTERCEPTOR(int, pthread_create, void *thread, void *attr, + void *(*callback)(void *), void *param) { EnsureMainThreadIDIsCorrect(); ScopedTaggingDisabler tagging_disabler; - ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie( - GetPageSizeCached(), "pthread_create")); - A->callback = callback; - A->param = param; + bool detached = [attr]() { + int d = 0; + return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); + }(); + ThreadStartArg *A = (ThreadStartArg *)InternalAlloc(sizeof(ThreadStartArg)); ScopedBlockSignals block(&A->starting_sigset_); // ASAN uses the same approach to disable leaks from pthread_create. # if CAN_SANITIZE_LEAKS __lsan::ScopedInterceptorDisabler lsan_disabler; # endif - return REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A); + + int result; + hwasanThreadArgRetval().Create(detached, {callback, param}, [&]() -> uptr { + result = REAL(pthread_create)(thread, attr, &HwasanThreadStartFunc, A); + return result ? 0 : *(uptr *)(thread); + }); + if (result != 0) + InternalFree(A); + return result; +} + +INTERCEPTOR(int, pthread_join, void *thread, void **retval) { + int result; + hwasanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_join)(thread, retval); + return !result; + }); + return result; } -INTERCEPTOR(int, pthread_join, void *t, void **arg) { - return REAL(pthread_join)(t, arg); +INTERCEPTOR(int, pthread_detach, void *thread) { + int result; + hwasanThreadArgRetval().Detach((uptr)thread, [&]() { + result = REAL(pthread_detach)(thread); + return !result; + }); + return result; } +INTERCEPTOR(void, pthread_exit, void *retval) { + hwasanThreadArgRetval().Finish(GetThreadSelf(), retval); + REAL(pthread_exit)(retval); +} + +# if SANITIZER_GLIBC +INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) { + int result; + hwasanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_tryjoin_np)(thread, ret); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, + const struct timespec *abstime) { + int result; + hwasanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_timedjoin_np)(thread, ret, abstime); + return !result; + }); + return result; +} +# endif + DEFINE_REAL_PTHREAD_FUNCTIONS DEFINE_REAL(int, vfork) @@ -85,13 +343,13 @@ DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) // Get and/or change the set of blocked signals. extern "C" int sigprocmask(int __how, const __hw_sigset_t *__restrict __set, __hw_sigset_t *__restrict __oset); -#define SIG_BLOCK 0 -#define SIG_SETMASK 2 +# define SIG_BLOCK 0 +# define SIG_SETMASK 2 extern "C" int __sigjmp_save(__hw_sigjmp_buf env, int savemask) { env[0].__magic = kHwJmpBufMagic; env[0].__mask_was_saved = - (savemask && sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0, - &env[0].__saved_mask) == 0); + (savemask && + sigprocmask(SIG_BLOCK, (__hw_sigset_t *)0, &env[0].__saved_mask) == 0); return 0; } @@ -120,26 +378,27 @@ InternalLongjmp(__hw_register_buf env, int retval) { # if defined(__aarch64__) register long int retval_tmp asm("x1") = retval; register void *env_address asm("x0") = &env[0]; - asm volatile("ldp x19, x20, [%0, #0<<3];" - "ldp x21, x22, [%0, #2<<3];" - "ldp x23, x24, [%0, #4<<3];" - "ldp x25, x26, [%0, #6<<3];" - "ldp x27, x28, [%0, #8<<3];" - "ldp x29, x30, [%0, #10<<3];" - "ldp d8, d9, [%0, #14<<3];" - "ldp d10, d11, [%0, #16<<3];" - "ldp d12, d13, [%0, #18<<3];" - "ldp d14, d15, [%0, #20<<3];" - "ldr x5, [%0, #13<<3];" - "mov sp, x5;" - // Return the value requested to return through arguments. - // This should be in x1 given what we requested above. - "cmp %1, #0;" - "mov x0, #1;" - "csel x0, %1, x0, ne;" - "br x30;" - : "+r"(env_address) - : "r"(retval_tmp)); + asm volatile( + "ldp x19, x20, [%0, #0<<3];" + "ldp x21, x22, [%0, #2<<3];" + "ldp x23, x24, [%0, #4<<3];" + "ldp x25, x26, [%0, #6<<3];" + "ldp x27, x28, [%0, #8<<3];" + "ldp x29, x30, [%0, #10<<3];" + "ldp d8, d9, [%0, #14<<3];" + "ldp d10, d11, [%0, #16<<3];" + "ldp d12, d13, [%0, #18<<3];" + "ldp d14, d15, [%0, #20<<3];" + "ldr x5, [%0, #13<<3];" + "mov sp, x5;" + // Return the value requested to return through arguments. + // This should be in x1 given what we requested above. + "cmp %1, #0;" + "mov x0, #1;" + "csel x0, %1, x0, ne;" + "br x30;" + : "+r"(env_address) + : "r"(retval_tmp)); # elif defined(__x86_64__) register long int retval_tmp asm("%rsi") = retval; register void *env_address asm("%rdi") = &env[0]; @@ -215,8 +474,7 @@ INTERCEPTOR(void, siglongjmp, __hw_sigjmp_buf env, int val) { if (env[0].__mask_was_saved) // Restore the saved signal mask. - (void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask, - (__hw_sigset_t *)0); + (void)sigprocmask(SIG_SETMASK, &env[0].__saved_mask, (__hw_sigset_t *)0); InternalLongjmp(env[0].__jmpbuf, val); } @@ -238,8 +496,8 @@ INTERCEPTOR(void, longjmp, __hw_jmp_buf env, int val) { } InternalLongjmp(env[0].__jmpbuf, val); } -#undef SIG_BLOCK -#undef SIG_SETMASK +# undef SIG_BLOCK +# undef SIG_SETMASK # endif // HWASAN_WITH_INTERCEPTORS @@ -254,7 +512,7 @@ int OnExit() { return 0; } -} // namespace __hwasan +} // namespace __hwasan namespace __hwasan { @@ -262,19 +520,30 @@ void InitializeInterceptors() { static int inited = 0; CHECK_EQ(inited, 0); -#if HWASAN_WITH_INTERCEPTORS -#if defined(__linux__) +# if HWASAN_WITH_INTERCEPTORS + InitializeCommonInterceptors(); + + (void)(read_iovec); + (void)(write_iovec); + +# if defined(__linux__) INTERCEPT_FUNCTION(__libc_longjmp); INTERCEPT_FUNCTION(longjmp); INTERCEPT_FUNCTION(siglongjmp); INTERCEPT_FUNCTION(vfork); -#endif // __linux__ +# endif // __linux__ INTERCEPT_FUNCTION(pthread_create); INTERCEPT_FUNCTION(pthread_join); + INTERCEPT_FUNCTION(pthread_detach); + INTERCEPT_FUNCTION(pthread_exit); +# if SANITIZER_GLIBC + INTERCEPT_FUNCTION(pthread_tryjoin_np); + INTERCEPT_FUNCTION(pthread_timedjoin_np); +# endif # endif inited = 1; } -} // namespace __hwasan +} // namespace __hwasan #endif // #if !SANITIZER_FUCHSIA diff --git a/libsanitizer/hwasan/hwasan_interface_internal.h b/libsanitizer/hwasan/hwasan_interface_internal.h index d1ecbb592a2..e7804cc4903 100644 --- a/libsanitizer/hwasan/hwasan_interface_internal.h +++ b/libsanitizer/hwasan/hwasan_interface_internal.h @@ -77,6 +77,32 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_load16_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_loadN_match_all(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16_match_all(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_loadN_match_all_noabort(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load1_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load2_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load4_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load8_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_load16_match_all_noabort(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_storeN(uptr, uptr); SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store1(uptr); @@ -103,6 +129,32 @@ SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_store16_noabort(uptr); SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_storeN_match_all(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8_match_all(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16_match_all(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_storeN_match_all_noabort(uptr, uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store1_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store2_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store4_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store8_match_all_noabort(uptr, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void __hwasan_store16_match_all_noabort(uptr, u8); + +SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_tag_memory(uptr p, u8 tag, uptr sz); SANITIZER_INTERFACE_ATTRIBUTE @@ -184,6 +236,13 @@ SANITIZER_INTERFACE_ATTRIBUTE void *__hwasan_memmove(void *dest, const void *src, uptr n); SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memcpy_match_all(void *dst, const void *src, uptr size, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memset_match_all(void *s, int c, uptr n, u8); +SANITIZER_INTERFACE_ATTRIBUTE +void *__hwasan_memmove_match_all(void *dest, const void *src, uptr n, u8); + +SANITIZER_INTERFACE_ATTRIBUTE void __hwasan_set_error_report_callback(void (*callback)(const char *)); } // extern "C" diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp index d3e4b5390e8..6f5e9432974 100644 --- a/libsanitizer/hwasan/hwasan_linux.cpp +++ b/libsanitizer/hwasan/hwasan_linux.cpp @@ -283,7 +283,7 @@ void InitThreads() { bool MemIsApp(uptr p) { // Memory outside the alias range has non-zero tags. # if !defined(HWASAN_ALIASING_MODE) - CHECK(GetTagFromPointer(p) == 0); + CHECK_EQ(GetTagFromPointer(p), 0); # endif return (p >= kHighMemStart && p <= kHighMemEnd) || @@ -302,8 +302,15 @@ extern "C" void __hwasan_thread_exit() { Thread *t = GetCurrentThread(); // Make sure that signal handler can not see a stale current thread pointer. atomic_signal_fence(memory_order_seq_cst); - if (t) + if (t) { + // Block async signals on the thread as the handler can be instrumented. + // After this point instrumented code can't access essential data from TLS + // and will crash. + // Bionic already calls __hwasan_thread_exit with blocked signals. + if (SANITIZER_GLIBC) + BlockSignals(); hwasanThreadList().ReleaseThread(t); + } } # if HWASAN_WITH_INTERCEPTORS diff --git a/libsanitizer/hwasan/hwasan_memintrinsics.cpp b/libsanitizer/hwasan/hwasan_memintrinsics.cpp index ea7f5ce40b0..16d6f908592 100644 --- a/libsanitizer/hwasan/hwasan_memintrinsics.cpp +++ b/libsanitizer/hwasan/hwasan_memintrinsics.cpp @@ -42,3 +42,33 @@ void *__hwasan_memmove(void *to, const void *from, uptr size) { reinterpret_cast<uptr>(from), size); return memmove(to, from, size); } + +void *__hwasan_memset_match_all(void *block, int c, uptr size, + u8 match_all_tag) { + if (GetTagFromPointer(reinterpret_cast<uptr>(block)) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(block), size); + return memset(block, c, size); +} + +void *__hwasan_memcpy_match_all(void *to, const void *from, uptr size, + u8 match_all_tag) { + if (GetTagFromPointer(reinterpret_cast<uptr>(to)) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(to), size); + if (GetTagFromPointer(reinterpret_cast<uptr>(from)) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Load>( + reinterpret_cast<uptr>(from), size); + return memcpy(to, from, size); +} + +void *__hwasan_memmove_match_all(void *to, const void *from, uptr size, + u8 match_all_tag) { + if (GetTagFromPointer(reinterpret_cast<uptr>(to)) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Store>( + reinterpret_cast<uptr>(to), size); + if (GetTagFromPointer(reinterpret_cast<uptr>(from)) != match_all_tag) + CheckAddressSized<ErrorAction::Recover, AccessType::Load>( + reinterpret_cast<uptr>(from), size); + return memmove(to, from, size); +} diff --git a/libsanitizer/hwasan/hwasan_platform_interceptors.h b/libsanitizer/hwasan/hwasan_platform_interceptors.h new file mode 100644 index 00000000000..d92b5105219 --- /dev/null +++ b/libsanitizer/hwasan/hwasan_platform_interceptors.h @@ -0,0 +1,1001 @@ +#ifndef HWASAN_PLATFORM_INTERCEPTORS_H +#define HWASAN_PLATFORM_INTERCEPTORS_H + +#include "sanitizer_common/sanitizer_platform_interceptors.h" + +// This file cancels out most of the sanitizer_common interception, thus +// allowing HWASan to selectively reuse some of the interceptors. +// +// To re-enable sanitizer_common's interception of a function, comment out +// the corresponding '#undef SANITIZER_INTERCEPT_fn' and +// '#define SANITIZER_INTERCEPT_fn 0': +// - We prefer to comment out rather than delete the lines, to show that +// it is deliberate, rather than an accidental omission. +// - We do not use '#define SANITIZE_INTERCEPT_fn 1', because +// interception is usually conditional (e.g., based on SI_POSIX); we let +// the condition in sanitizers_platform_interceptors.h take effect. + +// Originally generated with: +// cat ../sanitizer_common/sanitizer_platform_interceptors.h | grep '^#define SANITIZER_INTERCEPT' | cut -d ' ' -f 2 | while read x; do echo "#undef $x"; echo "#define $x 0"; echo; done +#undef SANITIZER_INTERCEPT_STRLEN +#define SANITIZER_INTERCEPT_STRLEN 0 + +#undef SANITIZER_INTERCEPT_STRNLEN +#define SANITIZER_INTERCEPT_STRNLEN 0 + +#undef SANITIZER_INTERCEPT_STRCMP +#define SANITIZER_INTERCEPT_STRCMP 0 + +#undef SANITIZER_INTERCEPT_STRSTR +#define SANITIZER_INTERCEPT_STRSTR 0 + +#undef SANITIZER_INTERCEPT_STRCASESTR +#define SANITIZER_INTERCEPT_STRCASESTR 0 + +#undef SANITIZER_INTERCEPT_STRTOK +#define SANITIZER_INTERCEPT_STRTOK 0 + +#undef SANITIZER_INTERCEPT_STRCHR +#define SANITIZER_INTERCEPT_STRCHR 0 + +#undef SANITIZER_INTERCEPT_STRCHRNUL +#define SANITIZER_INTERCEPT_STRCHRNUL 0 + +#undef SANITIZER_INTERCEPT_STRRCHR +#define SANITIZER_INTERCEPT_STRRCHR 0 + +#undef SANITIZER_INTERCEPT_STRSPN +#define SANITIZER_INTERCEPT_STRSPN 0 + +#undef SANITIZER_INTERCEPT_STRPBRK +#define SANITIZER_INTERCEPT_STRPBRK 0 + +#undef SANITIZER_INTERCEPT_TEXTDOMAIN +#define SANITIZER_INTERCEPT_TEXTDOMAIN 0 + +#undef SANITIZER_INTERCEPT_STRCASECMP +#define SANITIZER_INTERCEPT_STRCASECMP 0 + +// #undef SANITIZER_INTERCEPT_MEMSET +// #define SANITIZER_INTERCEPT_MEMSET 0 + +// #undef SANITIZER_INTERCEPT_MEMMOVE +// #define SANITIZER_INTERCEPT_MEMMOVE 0 + +// #undef SANITIZER_INTERCEPT_MEMCPY +// #define SANITIZER_INTERCEPT_MEMCPY 0 + +// #undef SANITIZER_INTERCEPT_MEMCMP +// #define SANITIZER_INTERCEPT_MEMCMP 0 + +// #undef SANITIZER_INTERCEPT_BCMP +// #define SANITIZER_INTERCEPT_BCMP 0 + +#undef SANITIZER_INTERCEPT_STRNDUP +#define SANITIZER_INTERCEPT_STRNDUP 0 + +#undef SANITIZER_INTERCEPT___STRNDUP +#define SANITIZER_INTERCEPT___STRNDUP 0 + +#undef SANITIZER_INTERCEPT_MEMMEM +#define SANITIZER_INTERCEPT_MEMMEM 0 + +#undef SANITIZER_INTERCEPT_MEMCHR +#define SANITIZER_INTERCEPT_MEMCHR 0 + +#undef SANITIZER_INTERCEPT_MEMRCHR +#define SANITIZER_INTERCEPT_MEMRCHR 0 + +#undef SANITIZER_INTERCEPT_READ +#define SANITIZER_INTERCEPT_READ 0 + +#undef SANITIZER_INTERCEPT_PREAD +#define SANITIZER_INTERCEPT_PREAD 0 + +#undef SANITIZER_INTERCEPT_WRITE +#define SANITIZER_INTERCEPT_WRITE 0 + +#undef SANITIZER_INTERCEPT_PWRITE +#define SANITIZER_INTERCEPT_PWRITE 0 + +#undef SANITIZER_INTERCEPT_FREAD +#define SANITIZER_INTERCEPT_FREAD 0 + +#undef SANITIZER_INTERCEPT_FWRITE +#define SANITIZER_INTERCEPT_FWRITE 0 + +#undef SANITIZER_INTERCEPT_FGETS +#define SANITIZER_INTERCEPT_FGETS 0 + +#undef SANITIZER_INTERCEPT_FPUTS +#define SANITIZER_INTERCEPT_FPUTS 0 + +#undef SANITIZER_INTERCEPT_PUTS +#define SANITIZER_INTERCEPT_PUTS 0 + +#undef SANITIZER_INTERCEPT_PREAD64 +#define SANITIZER_INTERCEPT_PREAD64 0 + +#undef SANITIZER_INTERCEPT_PWRITE64 +#define SANITIZER_INTERCEPT_PWRITE64 0 + +#undef SANITIZER_INTERCEPT_READV +#define SANITIZER_INTERCEPT_READV 0 + +#undef SANITIZER_INTERCEPT_WRITEV +#define SANITIZER_INTERCEPT_WRITEV 0 + +#undef SANITIZER_INTERCEPT_PREADV +#define SANITIZER_INTERCEPT_PREADV 0 + +#undef SANITIZER_INTERCEPT_PWRITEV +#define SANITIZER_INTERCEPT_PWRITEV 0 + +#undef SANITIZER_INTERCEPT_PREADV64 +#define SANITIZER_INTERCEPT_PREADV64 0 + +#undef SANITIZER_INTERCEPT_PWRITEV64 +#define SANITIZER_INTERCEPT_PWRITEV64 0 + +#undef SANITIZER_INTERCEPT_PRCTL +#define SANITIZER_INTERCEPT_PRCTL 0 + +#undef SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS +#define SANITIZER_INTERCEPT_LOCALTIME_AND_FRIENDS 0 + +#undef SANITIZER_INTERCEPT_STRPTIME +#define SANITIZER_INTERCEPT_STRPTIME 0 + +#undef SANITIZER_INTERCEPT_SCANF +#define SANITIZER_INTERCEPT_SCANF 0 + +#undef SANITIZER_INTERCEPT_ISOC99_SCANF +#define SANITIZER_INTERCEPT_ISOC99_SCANF 0 + +#undef SANITIZER_INTERCEPT_PRINTF +#define SANITIZER_INTERCEPT_PRINTF 0 + +#undef SANITIZER_INTERCEPT_PRINTF_L +#define SANITIZER_INTERCEPT_PRINTF_L 0 + +#undef SANITIZER_INTERCEPT_ISOC99_PRINTF +#define SANITIZER_INTERCEPT_ISOC99_PRINTF 0 + +#undef SANITIZER_INTERCEPT___PRINTF_CHK +#define SANITIZER_INTERCEPT___PRINTF_CHK 0 + +#undef SANITIZER_INTERCEPT_FREXP +#define SANITIZER_INTERCEPT_FREXP 0 + +#undef SANITIZER_INTERCEPT_FREXPF_FREXPL +#define SANITIZER_INTERCEPT_FREXPF_FREXPL 0 + +#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS +#define SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS 0 + +#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS +#define SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS 0 + +#undef SANITIZER_INTERCEPT_GETPWENT +#define SANITIZER_INTERCEPT_GETPWENT 0 + +#undef SANITIZER_INTERCEPT_FGETGRENT_R +#define SANITIZER_INTERCEPT_FGETGRENT_R 0 + +#undef SANITIZER_INTERCEPT_FGETPWENT +#define SANITIZER_INTERCEPT_FGETPWENT 0 + +#undef SANITIZER_INTERCEPT_GETPWENT_R +#define SANITIZER_INTERCEPT_GETPWENT_R 0 + +#undef SANITIZER_INTERCEPT_FGETPWENT_R +#define SANITIZER_INTERCEPT_FGETPWENT_R 0 + +#undef SANITIZER_INTERCEPT_SETPWENT +#define SANITIZER_INTERCEPT_SETPWENT 0 + +#undef SANITIZER_INTERCEPT_CLOCK_GETTIME +#define SANITIZER_INTERCEPT_CLOCK_GETTIME 0 + +#undef SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID +#define SANITIZER_INTERCEPT_CLOCK_GETCPUCLOCKID 0 + +#undef SANITIZER_INTERCEPT_GETITIMER +#define SANITIZER_INTERCEPT_GETITIMER 0 + +#undef SANITIZER_INTERCEPT_TIME +#define SANITIZER_INTERCEPT_TIME 0 + +#undef SANITIZER_INTERCEPT_GLOB +#define SANITIZER_INTERCEPT_GLOB 0 + +#undef SANITIZER_INTERCEPT_GLOB64 +#define SANITIZER_INTERCEPT_GLOB64 0 + +#undef SANITIZER_INTERCEPT___B64_TO +#define SANITIZER_INTERCEPT___B64_TO 0 + +#undef SANITIZER_INTERCEPT_DN_COMP_EXPAND +#define SANITIZER_INTERCEPT_DN_COMP_EXPAND 0 + +#undef SANITIZER_INTERCEPT_POSIX_SPAWN +#define SANITIZER_INTERCEPT_POSIX_SPAWN 0 + +#undef SANITIZER_INTERCEPT_WAIT +#define SANITIZER_INTERCEPT_WAIT 0 + +#undef SANITIZER_INTERCEPT_INET +#define SANITIZER_INTERCEPT_INET 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM +#define SANITIZER_INTERCEPT_PTHREAD_GETSCHEDPARAM 0 + +#undef SANITIZER_INTERCEPT_GETADDRINFO +#define SANITIZER_INTERCEPT_GETADDRINFO 0 + +#undef SANITIZER_INTERCEPT_GETNAMEINFO +#define SANITIZER_INTERCEPT_GETNAMEINFO 0 + +#undef SANITIZER_INTERCEPT_GETSOCKNAME +#define SANITIZER_INTERCEPT_GETSOCKNAME 0 + +#undef SANITIZER_INTERCEPT_GETHOSTBYNAME +#define SANITIZER_INTERCEPT_GETHOSTBYNAME 0 + +#undef SANITIZER_INTERCEPT_GETHOSTBYNAME2 +#define SANITIZER_INTERCEPT_GETHOSTBYNAME2 0 + +#undef SANITIZER_INTERCEPT_GETHOSTBYNAME_R +#define SANITIZER_INTERCEPT_GETHOSTBYNAME_R 0 + +#undef SANITIZER_INTERCEPT_GETHOSTBYNAME2_R +#define SANITIZER_INTERCEPT_GETHOSTBYNAME2_R 0 + +#undef SANITIZER_INTERCEPT_GETHOSTBYADDR_R +#define SANITIZER_INTERCEPT_GETHOSTBYADDR_R 0 + +#undef SANITIZER_INTERCEPT_GETHOSTENT_R +#define SANITIZER_INTERCEPT_GETHOSTENT_R 0 + +#undef SANITIZER_INTERCEPT_GETSOCKOPT +#define SANITIZER_INTERCEPT_GETSOCKOPT 0 + +#undef SANITIZER_INTERCEPT_ACCEPT +#define SANITIZER_INTERCEPT_ACCEPT 0 + +#undef SANITIZER_INTERCEPT_ACCEPT4 +#define SANITIZER_INTERCEPT_ACCEPT4 0 + +#undef SANITIZER_INTERCEPT_PACCEPT +#define SANITIZER_INTERCEPT_PACCEPT 0 + +#undef SANITIZER_INTERCEPT_MODF +#define SANITIZER_INTERCEPT_MODF 0 + +#undef SANITIZER_INTERCEPT_RECVMSG +#define SANITIZER_INTERCEPT_RECVMSG 0 + +#undef SANITIZER_INTERCEPT_SENDMSG +#define SANITIZER_INTERCEPT_SENDMSG 0 + +#undef SANITIZER_INTERCEPT_RECVMMSG +#define SANITIZER_INTERCEPT_RECVMMSG 0 + +#undef SANITIZER_INTERCEPT_SENDMMSG +#define SANITIZER_INTERCEPT_SENDMMSG 0 + +#undef SANITIZER_INTERCEPT_SYSMSG +#define SANITIZER_INTERCEPT_SYSMSG 0 + +#undef SANITIZER_INTERCEPT_GETPEERNAME +#define SANITIZER_INTERCEPT_GETPEERNAME 0 + +#undef SANITIZER_INTERCEPT_IOCTL +#define SANITIZER_INTERCEPT_IOCTL 0 + +#undef SANITIZER_INTERCEPT_INET_ATON +#define SANITIZER_INTERCEPT_INET_ATON 0 + +#undef SANITIZER_INTERCEPT_SYSINFO +#define SANITIZER_INTERCEPT_SYSINFO 0 + +#undef SANITIZER_INTERCEPT_READDIR +#define SANITIZER_INTERCEPT_READDIR 0 + +#undef SANITIZER_INTERCEPT_READDIR64 +#define SANITIZER_INTERCEPT_READDIR64 0 + +#undef SANITIZER_INTERCEPT_PTRACE +#define SANITIZER_INTERCEPT_PTRACE 0 + +#undef SANITIZER_INTERCEPT_PTRACE +#define SANITIZER_INTERCEPT_PTRACE 0 + +#undef SANITIZER_INTERCEPT_SETLOCALE +#define SANITIZER_INTERCEPT_SETLOCALE 0 + +#undef SANITIZER_INTERCEPT_GETCWD +#define SANITIZER_INTERCEPT_GETCWD 0 + +#undef SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME +#define SANITIZER_INTERCEPT_GET_CURRENT_DIR_NAME 0 + +#undef SANITIZER_INTERCEPT_STRTOIMAX +#define SANITIZER_INTERCEPT_STRTOIMAX 0 + +#undef SANITIZER_INTERCEPT_MBSTOWCS +#define SANITIZER_INTERCEPT_MBSTOWCS 0 + +#undef SANITIZER_INTERCEPT_MBSNRTOWCS +#define SANITIZER_INTERCEPT_MBSNRTOWCS 0 + +#undef SANITIZER_INTERCEPT_WCSTOMBS +#define SANITIZER_INTERCEPT_WCSTOMBS 0 + +#undef SANITIZER_INTERCEPT_STRXFRM +#define SANITIZER_INTERCEPT_STRXFRM 0 + +#undef SANITIZER_INTERCEPT___STRXFRM_L +#define SANITIZER_INTERCEPT___STRXFRM_L 0 + +#undef SANITIZER_INTERCEPT_WCSXFRM +#define SANITIZER_INTERCEPT_WCSXFRM 0 + +#undef SANITIZER_INTERCEPT___WCSXFRM_L +#define SANITIZER_INTERCEPT___WCSXFRM_L 0 + +#undef SANITIZER_INTERCEPT_WCSNRTOMBS +#define SANITIZER_INTERCEPT_WCSNRTOMBS 0 + +#undef SANITIZER_INTERCEPT_WCRTOMB +#define SANITIZER_INTERCEPT_WCRTOMB 0 + +#undef SANITIZER_INTERCEPT_WCTOMB +#define SANITIZER_INTERCEPT_WCTOMB 0 + +#undef SANITIZER_INTERCEPT_TCGETATTR +#define SANITIZER_INTERCEPT_TCGETATTR 0 + +#undef SANITIZER_INTERCEPT_REALPATH +#define SANITIZER_INTERCEPT_REALPATH 0 + +#undef SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME +#define SANITIZER_INTERCEPT_CANONICALIZE_FILE_NAME 0 + +#undef SANITIZER_INTERCEPT_CONFSTR +#define SANITIZER_INTERCEPT_CONFSTR 0 + +#undef SANITIZER_INTERCEPT_SCHED_GETAFFINITY +#define SANITIZER_INTERCEPT_SCHED_GETAFFINITY 0 + +#undef SANITIZER_INTERCEPT_SCHED_GETPARAM +#define SANITIZER_INTERCEPT_SCHED_GETPARAM 0 + +#undef SANITIZER_INTERCEPT_STRERROR +#define SANITIZER_INTERCEPT_STRERROR 0 + +#undef SANITIZER_INTERCEPT_STRERROR_R +#define SANITIZER_INTERCEPT_STRERROR_R 0 + +#undef SANITIZER_INTERCEPT_XPG_STRERROR_R +#define SANITIZER_INTERCEPT_XPG_STRERROR_R 0 + +#undef SANITIZER_INTERCEPT_SCANDIR +#define SANITIZER_INTERCEPT_SCANDIR 0 + +#undef SANITIZER_INTERCEPT_SCANDIR64 +#define SANITIZER_INTERCEPT_SCANDIR64 0 + +#undef SANITIZER_INTERCEPT_GETGROUPS +#define SANITIZER_INTERCEPT_GETGROUPS 0 + +#undef SANITIZER_INTERCEPT_POLL +#define SANITIZER_INTERCEPT_POLL 0 + +#undef SANITIZER_INTERCEPT_PPOLL +#define SANITIZER_INTERCEPT_PPOLL 0 + +#undef SANITIZER_INTERCEPT_WORDEXP +#define SANITIZER_INTERCEPT_WORDEXP 0 + +#undef SANITIZER_INTERCEPT_SIGWAIT +#define SANITIZER_INTERCEPT_SIGWAIT 0 + +#undef SANITIZER_INTERCEPT_SIGWAITINFO +#define SANITIZER_INTERCEPT_SIGWAITINFO 0 + +#undef SANITIZER_INTERCEPT_SIGTIMEDWAIT +#define SANITIZER_INTERCEPT_SIGTIMEDWAIT 0 + +#undef SANITIZER_INTERCEPT_SIGSETOPS +#define SANITIZER_INTERCEPT_SIGSETOPS 0 + +#undef SANITIZER_INTERCEPT_SIGSET_LOGICOPS +#define SANITIZER_INTERCEPT_SIGSET_LOGICOPS 0 + +#undef SANITIZER_INTERCEPT_SIGPENDING +#define SANITIZER_INTERCEPT_SIGPENDING 0 + +#undef SANITIZER_INTERCEPT_SIGPROCMASK +#define SANITIZER_INTERCEPT_SIGPROCMASK 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK +#define SANITIZER_INTERCEPT_PTHREAD_SIGMASK 0 + +#undef SANITIZER_INTERCEPT_BACKTRACE +#define SANITIZER_INTERCEPT_BACKTRACE 0 + +#undef SANITIZER_INTERCEPT_GETMNTENT +#define SANITIZER_INTERCEPT_GETMNTENT 0 + +#undef SANITIZER_INTERCEPT_GETMNTENT_R +#define SANITIZER_INTERCEPT_GETMNTENT_R 0 + +#undef SANITIZER_INTERCEPT_STATFS +#define SANITIZER_INTERCEPT_STATFS 0 + +#undef SANITIZER_INTERCEPT_STATFS64 +#define SANITIZER_INTERCEPT_STATFS64 0 + +#undef SANITIZER_INTERCEPT_STATVFS +#define SANITIZER_INTERCEPT_STATVFS 0 + +#undef SANITIZER_INTERCEPT_STATVFS64 +#define SANITIZER_INTERCEPT_STATVFS64 0 + +#undef SANITIZER_INTERCEPT_INITGROUPS +#define SANITIZER_INTERCEPT_INITGROUPS 0 + +#undef SANITIZER_INTERCEPT_ETHER_NTOA_ATON +#define SANITIZER_INTERCEPT_ETHER_NTOA_ATON 0 + +#undef SANITIZER_INTERCEPT_ETHER_HOST +#define SANITIZER_INTERCEPT_ETHER_HOST 0 + +#undef SANITIZER_INTERCEPT_ETHER_R +#define SANITIZER_INTERCEPT_ETHER_R 0 + +#undef SANITIZER_INTERCEPT_SHMCTL +#define SANITIZER_INTERCEPT_SHMCTL 0 + +#undef SANITIZER_INTERCEPT_RANDOM_R +#define SANITIZER_INTERCEPT_RANDOM_R 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_ATTR_GET +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP +#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED +#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETTYPE 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPROTOCOL 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPRIOCEILING 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP +#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETROBUST_NP 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED +#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETPSHARED 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP +#define SANITIZER_INTERCEPT_PTHREAD_RWLOCKATTR_GETKIND_NP 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETPSHARED 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK +#define SANITIZER_INTERCEPT_PTHREAD_CONDATTR_GETCLOCK 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED +#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED 0 + +#undef SANITIZER_INTERCEPT_TRYJOIN +#define SANITIZER_INTERCEPT_TRYJOIN 0 + +#undef SANITIZER_INTERCEPT_TIMEDJOIN +#define SANITIZER_INTERCEPT_TIMEDJOIN 0 + +#undef SANITIZER_INTERCEPT_THR_EXIT +#define SANITIZER_INTERCEPT_THR_EXIT 0 + +#undef SANITIZER_INTERCEPT_TMPNAM +#define SANITIZER_INTERCEPT_TMPNAM 0 + +#undef SANITIZER_INTERCEPT_TMPNAM_R +#define SANITIZER_INTERCEPT_TMPNAM_R 0 + +#undef SANITIZER_INTERCEPT_PTSNAME +#define SANITIZER_INTERCEPT_PTSNAME 0 + +#undef SANITIZER_INTERCEPT_PTSNAME_R +#define SANITIZER_INTERCEPT_PTSNAME_R 0 + +#undef SANITIZER_INTERCEPT_TTYNAME +#define SANITIZER_INTERCEPT_TTYNAME 0 + +#undef SANITIZER_INTERCEPT_TTYNAME_R +#define SANITIZER_INTERCEPT_TTYNAME_R 0 + +#undef SANITIZER_INTERCEPT_TEMPNAM +#define SANITIZER_INTERCEPT_TEMPNAM 0 + +#undef SANITIZER_INTERCEPT_SINCOS +#define SANITIZER_INTERCEPT_SINCOS 0 + +#undef SANITIZER_INTERCEPT_REMQUO +#define SANITIZER_INTERCEPT_REMQUO 0 + +#undef SANITIZER_INTERCEPT_REMQUOL +#define SANITIZER_INTERCEPT_REMQUOL 0 + +#undef SANITIZER_INTERCEPT_LGAMMA +#define SANITIZER_INTERCEPT_LGAMMA 0 + +#undef SANITIZER_INTERCEPT_LGAMMAL +#define SANITIZER_INTERCEPT_LGAMMAL 0 + +#undef SANITIZER_INTERCEPT_LGAMMA_R +#define SANITIZER_INTERCEPT_LGAMMA_R 0 + +#undef SANITIZER_INTERCEPT_LGAMMAL_R +#define SANITIZER_INTERCEPT_LGAMMAL_R 0 + +#undef SANITIZER_INTERCEPT_DRAND48_R +#define SANITIZER_INTERCEPT_DRAND48_R 0 + +#undef SANITIZER_INTERCEPT_RAND_R +#define SANITIZER_INTERCEPT_RAND_R 0 + +#undef SANITIZER_INTERCEPT_ICONV +#define SANITIZER_INTERCEPT_ICONV 0 + +#undef SANITIZER_INTERCEPT_TIMES +#define SANITIZER_INTERCEPT_TIMES 0 + +#undef SANITIZER_INTERCEPT_GETLINE +#define SANITIZER_INTERCEPT_GETLINE 0 + +#undef SANITIZER_INTERCEPT__EXIT +#define SANITIZER_INTERCEPT__EXIT 0 + +#undef SANITIZER_INTERCEPT___LIBC_MUTEX +#define SANITIZER_INTERCEPT___LIBC_MUTEX 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP +#define SANITIZER_INTERCEPT_PTHREAD_SETNAME_NP 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP +#define SANITIZER_INTERCEPT_PTHREAD_GETNAME_NP 0 + +#undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#define SANITIZER_INTERCEPT_TLS_GET_ADDR 0 + +#undef SANITIZER_INTERCEPT_LISTXATTR +#define SANITIZER_INTERCEPT_LISTXATTR 0 + +#undef SANITIZER_INTERCEPT_GETXATTR +#define SANITIZER_INTERCEPT_GETXATTR 0 + +#undef SANITIZER_INTERCEPT_GETRESID +#define SANITIZER_INTERCEPT_GETRESID 0 + +#undef SANITIZER_INTERCEPT_GETIFADDRS +#define SANITIZER_INTERCEPT_GETIFADDRS 0 + +#undef SANITIZER_INTERCEPT_IF_INDEXTONAME +#define SANITIZER_INTERCEPT_IF_INDEXTONAME 0 + +#undef SANITIZER_INTERCEPT_CAPGET +#define SANITIZER_INTERCEPT_CAPGET 0 + +#undef SANITIZER_INTERCEPT_AEABI_MEM +#define SANITIZER_INTERCEPT_AEABI_MEM 0 + +#undef SANITIZER_INTERCEPT_AEABI_MEM +#define SANITIZER_INTERCEPT_AEABI_MEM 0 + +#undef SANITIZER_INTERCEPT___BZERO +#define SANITIZER_INTERCEPT___BZERO 0 + +#undef SANITIZER_INTERCEPT_BZERO +#define SANITIZER_INTERCEPT_BZERO 0 + +#undef SANITIZER_INTERCEPT_FTIME +#define SANITIZER_INTERCEPT_FTIME 0 + +#undef SANITIZER_INTERCEPT_XDR +#define SANITIZER_INTERCEPT_XDR 0 + +#undef SANITIZER_INTERCEPT_XDRREC +#define SANITIZER_INTERCEPT_XDRREC 0 + +#undef SANITIZER_INTERCEPT_TSEARCH +#define SANITIZER_INTERCEPT_TSEARCH 0 + +#undef SANITIZER_INTERCEPT_LIBIO_INTERNALS +#define SANITIZER_INTERCEPT_LIBIO_INTERNALS 0 + +#undef SANITIZER_INTERCEPT_FOPEN +#define SANITIZER_INTERCEPT_FOPEN 0 + +#undef SANITIZER_INTERCEPT_FOPEN64 +#define SANITIZER_INTERCEPT_FOPEN64 0 + +#undef SANITIZER_INTERCEPT_OPEN_MEMSTREAM +#define SANITIZER_INTERCEPT_OPEN_MEMSTREAM 0 + +#undef SANITIZER_INTERCEPT_OBSTACK +#define SANITIZER_INTERCEPT_OBSTACK 0 + +#undef SANITIZER_INTERCEPT_FFLUSH +#define SANITIZER_INTERCEPT_FFLUSH 0 + +#undef SANITIZER_INTERCEPT_FCLOSE +#define SANITIZER_INTERCEPT_FCLOSE 0 + +#undef SANITIZER_INTERCEPT_DLOPEN_DLCLOSE +#define SANITIZER_INTERCEPT_DLOPEN_DLCLOSE 0 + +#undef SANITIZER_INTERCEPT_GETPASS +#define SANITIZER_INTERCEPT_GETPASS 0 + +#undef SANITIZER_INTERCEPT_TIMERFD +#define SANITIZER_INTERCEPT_TIMERFD 0 + +#undef SANITIZER_INTERCEPT_MLOCKX +#define SANITIZER_INTERCEPT_MLOCKX 0 + +#undef SANITIZER_INTERCEPT_FOPENCOOKIE +#define SANITIZER_INTERCEPT_FOPENCOOKIE 0 + +#undef SANITIZER_INTERCEPT_SEM +#define SANITIZER_INTERCEPT_SEM 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_SETCANCEL +#define SANITIZER_INTERCEPT_PTHREAD_SETCANCEL 0 + +#undef SANITIZER_INTERCEPT_MINCORE +#define SANITIZER_INTERCEPT_MINCORE 0 + +#undef SANITIZER_INTERCEPT_PROCESS_VM_READV +#define SANITIZER_INTERCEPT_PROCESS_VM_READV 0 + +#undef SANITIZER_INTERCEPT_CTERMID +#define SANITIZER_INTERCEPT_CTERMID 0 + +#undef SANITIZER_INTERCEPT_CTERMID_R +#define SANITIZER_INTERCEPT_CTERMID_R 0 + +#undef SANITIZER_INTERCEPTOR_HOOKS +#define SANITIZER_INTERCEPTOR_HOOKS 0 + +#undef SANITIZER_INTERCEPT_RECV_RECVFROM +#define SANITIZER_INTERCEPT_RECV_RECVFROM 0 + +#undef SANITIZER_INTERCEPT_SEND_SENDTO +#define SANITIZER_INTERCEPT_SEND_SENDTO 0 + +#undef SANITIZER_INTERCEPT_EVENTFD_READ_WRITE +#define SANITIZER_INTERCEPT_EVENTFD_READ_WRITE 0 + +#undef SANITIZER_INTERCEPT_STAT +#define SANITIZER_INTERCEPT_STAT 0 + +#undef SANITIZER_INTERCEPT_STAT64 +#define SANITIZER_INTERCEPT_STAT64 0 + +#undef SANITIZER_INTERCEPT_LSTAT +#define SANITIZER_INTERCEPT_LSTAT 0 + +#undef SANITIZER_INTERCEPT___XSTAT +#define SANITIZER_INTERCEPT___XSTAT 0 + +#undef SANITIZER_INTERCEPT___XSTAT64 +#define SANITIZER_INTERCEPT___XSTAT64 0 + +#undef SANITIZER_INTERCEPT___LXSTAT +#define SANITIZER_INTERCEPT___LXSTAT 0 + +#undef SANITIZER_INTERCEPT___LXSTAT64 +#define SANITIZER_INTERCEPT___LXSTAT64 0 + +#undef SANITIZER_INTERCEPT_UTMP +#define SANITIZER_INTERCEPT_UTMP 0 + +#undef SANITIZER_INTERCEPT_UTMPX +#define SANITIZER_INTERCEPT_UTMPX 0 + +#undef SANITIZER_INTERCEPT_GETLOADAVG +#define SANITIZER_INTERCEPT_GETLOADAVG 0 + +// #undef SANITIZER_INTERCEPT_MMAP +// #define SANITIZER_INTERCEPT_MMAP 0 + +#undef SANITIZER_INTERCEPT_MMAP64 +#define SANITIZER_INTERCEPT_MMAP64 0 + +#undef SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO +#define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO 0 + +#undef SANITIZER_INTERCEPT_MEMALIGN +#define SANITIZER_INTERCEPT_MEMALIGN 0 + +#undef SANITIZER_INTERCEPT___LIBC_MEMALIGN +#define SANITIZER_INTERCEPT___LIBC_MEMALIGN 0 + +#undef SANITIZER_INTERCEPT_PVALLOC +#define SANITIZER_INTERCEPT_PVALLOC 0 + +#undef SANITIZER_INTERCEPT_CFREE +#define SANITIZER_INTERCEPT_CFREE 0 + +#undef SANITIZER_INTERCEPT_REALLOCARRAY +#define SANITIZER_INTERCEPT_REALLOCARRAY 0 + +#undef SANITIZER_INTERCEPT_ALIGNED_ALLOC +#define SANITIZER_INTERCEPT_ALIGNED_ALLOC 0 + +#undef SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE +#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE 0 + +#undef SANITIZER_INTERCEPT_MCHECK_MPROBE +#define SANITIZER_INTERCEPT_MCHECK_MPROBE 0 + +#undef SANITIZER_INTERCEPT_WCSLEN +#define SANITIZER_INTERCEPT_WCSLEN 0 + +#undef SANITIZER_INTERCEPT_WCSCAT +#define SANITIZER_INTERCEPT_WCSCAT 0 + +#undef SANITIZER_INTERCEPT_WCSDUP +#define SANITIZER_INTERCEPT_WCSDUP 0 + +#undef SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION +#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION 0 + +#undef SANITIZER_INTERCEPT_BSD_SIGNAL +#define SANITIZER_INTERCEPT_BSD_SIGNAL 0 + +#undef SANITIZER_INTERCEPT_ACCT +#define SANITIZER_INTERCEPT_ACCT 0 + +#undef SANITIZER_INTERCEPT_USER_FROM_UID +#define SANITIZER_INTERCEPT_USER_FROM_UID 0 + +#undef SANITIZER_INTERCEPT_UID_FROM_USER +#define SANITIZER_INTERCEPT_UID_FROM_USER 0 + +#undef SANITIZER_INTERCEPT_GROUP_FROM_GID +#define SANITIZER_INTERCEPT_GROUP_FROM_GID 0 + +#undef SANITIZER_INTERCEPT_GID_FROM_GROUP +#define SANITIZER_INTERCEPT_GID_FROM_GROUP 0 + +#undef SANITIZER_INTERCEPT_ACCESS +#define SANITIZER_INTERCEPT_ACCESS 0 + +#undef SANITIZER_INTERCEPT_FACCESSAT +#define SANITIZER_INTERCEPT_FACCESSAT 0 + +#undef SANITIZER_INTERCEPT_GETGROUPLIST +#define SANITIZER_INTERCEPT_GETGROUPLIST 0 + +#undef SANITIZER_INTERCEPT_STRLCPY +#define SANITIZER_INTERCEPT_STRLCPY 0 + +#undef SANITIZER_INTERCEPT_NAME_TO_HANDLE_AT +#define SANITIZER_INTERCEPT_NAME_TO_HANDLE_AT 0 + +#undef SANITIZER_INTERCEPT_OPEN_BY_HANDLE_AT +#define SANITIZER_INTERCEPT_OPEN_BY_HANDLE_AT 0 + +#undef SANITIZER_INTERCEPT_READLINK +#define SANITIZER_INTERCEPT_READLINK 0 + +#undef SANITIZER_INTERCEPT_READLINKAT +#define SANITIZER_INTERCEPT_READLINKAT 0 + +#undef SANITIZER_INTERCEPT_DEVNAME +#define SANITIZER_INTERCEPT_DEVNAME 0 + +#undef SANITIZER_INTERCEPT_DEVNAME_R +#define SANITIZER_INTERCEPT_DEVNAME_R 0 + +#undef SANITIZER_INTERCEPT_FGETLN +#define SANITIZER_INTERCEPT_FGETLN 0 + +#undef SANITIZER_INTERCEPT_STRMODE +#define SANITIZER_INTERCEPT_STRMODE 0 + +#undef SANITIZER_INTERCEPT_TTYENT +#define SANITIZER_INTERCEPT_TTYENT 0 + +#undef SANITIZER_INTERCEPT_TTYENTPATH +#define SANITIZER_INTERCEPT_TTYENTPATH 0 + +#undef SANITIZER_INTERCEPT_PROTOENT +#define SANITIZER_INTERCEPT_PROTOENT 0 + +#undef SANITIZER_INTERCEPT_PROTOENT_R +#define SANITIZER_INTERCEPT_PROTOENT_R 0 + +#undef SANITIZER_INTERCEPT_NETENT +#define SANITIZER_INTERCEPT_NETENT 0 + +#undef SANITIZER_INTERCEPT_SETVBUF +#define SANITIZER_INTERCEPT_SETVBUF 0 + +#undef SANITIZER_INTERCEPT_GETMNTINFO +#define SANITIZER_INTERCEPT_GETMNTINFO 0 + +#undef SANITIZER_INTERCEPT_MI_VECTOR_HASH +#define SANITIZER_INTERCEPT_MI_VECTOR_HASH 0 + +#undef SANITIZER_INTERCEPT_GETVFSSTAT +#define SANITIZER_INTERCEPT_GETVFSSTAT 0 + +#undef SANITIZER_INTERCEPT_REGEX +#define SANITIZER_INTERCEPT_REGEX 0 + +#undef SANITIZER_INTERCEPT_REGEXSUB +#define SANITIZER_INTERCEPT_REGEXSUB 0 + +#undef SANITIZER_INTERCEPT_FTS +#define SANITIZER_INTERCEPT_FTS 0 + +#undef SANITIZER_INTERCEPT_SYSCTL +#define SANITIZER_INTERCEPT_SYSCTL 0 + +#undef SANITIZER_INTERCEPT_ASYSCTL +#define SANITIZER_INTERCEPT_ASYSCTL 0 + +#undef SANITIZER_INTERCEPT_SYSCTLGETMIBINFO +#define SANITIZER_INTERCEPT_SYSCTLGETMIBINFO 0 + +#undef SANITIZER_INTERCEPT_NL_LANGINFO +#define SANITIZER_INTERCEPT_NL_LANGINFO 0 + +#undef SANITIZER_INTERCEPT_MODCTL +#define SANITIZER_INTERCEPT_MODCTL 0 + +#undef SANITIZER_INTERCEPT_CAPSICUM +#define SANITIZER_INTERCEPT_CAPSICUM 0 + +#undef SANITIZER_INTERCEPT_STRTONUM +#define SANITIZER_INTERCEPT_STRTONUM 0 + +#undef SANITIZER_INTERCEPT_FPARSELN +#define SANITIZER_INTERCEPT_FPARSELN 0 + +#undef SANITIZER_INTERCEPT_STATVFS1 +#define SANITIZER_INTERCEPT_STATVFS1 0 + +#undef SANITIZER_INTERCEPT_STRTOI +#define SANITIZER_INTERCEPT_STRTOI 0 + +#undef SANITIZER_INTERCEPT_CAPSICUM +#define SANITIZER_INTERCEPT_CAPSICUM 0 + +#undef SANITIZER_INTERCEPT_SHA1 +#define SANITIZER_INTERCEPT_SHA1 0 + +#undef SANITIZER_INTERCEPT_MD4 +#define SANITIZER_INTERCEPT_MD4 0 + +#undef SANITIZER_INTERCEPT_RMD160 +#define SANITIZER_INTERCEPT_RMD160 0 + +#undef SANITIZER_INTERCEPT_MD5 +#define SANITIZER_INTERCEPT_MD5 0 + +#undef SANITIZER_INTERCEPT_FSEEK +#define SANITIZER_INTERCEPT_FSEEK 0 + +#undef SANITIZER_INTERCEPT_MD2 +#define SANITIZER_INTERCEPT_MD2 0 + +#undef SANITIZER_INTERCEPT_SHA2 +#define SANITIZER_INTERCEPT_SHA2 0 + +#undef SANITIZER_INTERCEPT_CDB +#define SANITIZER_INTERCEPT_CDB 0 + +#undef SANITIZER_INTERCEPT_VIS +#define SANITIZER_INTERCEPT_VIS 0 + +#undef SANITIZER_INTERCEPT_POPEN +#define SANITIZER_INTERCEPT_POPEN 0 + +#undef SANITIZER_INTERCEPT_POPENVE +#define SANITIZER_INTERCEPT_POPENVE 0 + +#undef SANITIZER_INTERCEPT_PCLOSE +#define SANITIZER_INTERCEPT_PCLOSE 0 + +#undef SANITIZER_INTERCEPT_FUNOPEN +#define SANITIZER_INTERCEPT_FUNOPEN 0 + +#undef SANITIZER_INTERCEPT_FUNOPEN2 +#define SANITIZER_INTERCEPT_FUNOPEN2 0 + +#undef SANITIZER_INTERCEPT_GETFSENT +#define SANITIZER_INTERCEPT_GETFSENT 0 + +#undef SANITIZER_INTERCEPT_ARC4RANDOM +#define SANITIZER_INTERCEPT_ARC4RANDOM 0 + +#undef SANITIZER_INTERCEPT_FDEVNAME +#define SANITIZER_INTERCEPT_FDEVNAME 0 + +#undef SANITIZER_INTERCEPT_GETUSERSHELL +#define SANITIZER_INTERCEPT_GETUSERSHELL 0 + +#undef SANITIZER_INTERCEPT_SL_INIT +#define SANITIZER_INTERCEPT_SL_INIT 0 + +#undef SANITIZER_INTERCEPT_GETRANDOM +#define SANITIZER_INTERCEPT_GETRANDOM 0 + +#undef SANITIZER_INTERCEPT___CXA_ATEXIT +#define SANITIZER_INTERCEPT___CXA_ATEXIT 0 + +#undef SANITIZER_INTERCEPT_ATEXIT +#define SANITIZER_INTERCEPT_ATEXIT 0 + +#undef SANITIZER_INTERCEPT_PTHREAD_ATFORK +#define SANITIZER_INTERCEPT_PTHREAD_ATFORK 0 + +#undef SANITIZER_INTERCEPT_GETENTROPY +#define SANITIZER_INTERCEPT_GETENTROPY 0 + +#undef SANITIZER_INTERCEPT_QSORT +#define SANITIZER_INTERCEPT_QSORT 0 + +#undef SANITIZER_INTERCEPT_QSORT_R +#define SANITIZER_INTERCEPT_QSORT_R 0 + +#undef SANITIZER_INTERCEPT_BSEARCH +#define SANITIZER_INTERCEPT_BSEARCH 0 + +#undef SANITIZER_INTERCEPT_SIGALTSTACK +#define SANITIZER_INTERCEPT_SIGALTSTACK 0 + +#undef SANITIZER_INTERCEPT_UNAME +#define SANITIZER_INTERCEPT_UNAME 0 + +#undef SANITIZER_INTERCEPT___XUNAME +#define SANITIZER_INTERCEPT___XUNAME 0 + +#undef SANITIZER_INTERCEPT_FLOPEN +#define SANITIZER_INTERCEPT_FLOPEN 0 + +#undef SANITIZER_INTERCEPT_PROCCTL +#define SANITIZER_INTERCEPT_PROCCTL 0 + +#undef SANITIZER_INTERCEPT_HEXDUMP +#define SANITIZER_INTERCEPT_HEXDUMP 0 + +#undef SANITIZER_INTERCEPT_ARGP_PARSE +#define SANITIZER_INTERCEPT_ARGP_PARSE 0 + +#endif // HWASAN_PLATFORM_INTERCEPTORS_H diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp index 8f9dc6cf13e..5e8aa315801 100644 --- a/libsanitizer/hwasan/hwasan_report.cpp +++ b/libsanitizer/hwasan/hwasan_report.cpp @@ -22,8 +22,10 @@ #include "hwasan_thread.h" #include "hwasan_thread_list.h" #include "sanitizer_common/sanitizer_allocator_internal.h" +#include "sanitizer_common/sanitizer_array_ref.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_mutex.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stackdepot.h" @@ -36,7 +38,7 @@ namespace __hwasan { class ScopedReport { public: - ScopedReport(bool fatal = false) : error_message_(1), fatal(fatal) { + explicit ScopedReport(bool fatal) : fatal(fatal) { Lock lock(&error_message_lock_); error_message_ptr_ = fatal ? &error_message_ : nullptr; ++hwasan_report_count; @@ -64,11 +66,7 @@ class ScopedReport { Lock lock(&error_message_lock_); if (!error_message_ptr_) return; - uptr len = internal_strlen(msg); - uptr old_size = error_message_ptr_->size(); - error_message_ptr_->resize(old_size + len); - // overwrite old trailing '\0', keep new trailing '\0' untouched. - internal_memcpy(&(*error_message_ptr_)[old_size - 1], msg, len); + error_message_ptr_->Append(msg); } static void SetErrorReportCallback(void (*callback)(const char *)) { @@ -77,17 +75,17 @@ class ScopedReport { } private: - ScopedErrorReportLock error_report_lock_; - InternalMmapVector<char> error_message_; + InternalScopedString error_message_; bool fatal; - static InternalMmapVector<char> *error_message_ptr_; static Mutex error_message_lock_; + static InternalScopedString *error_message_ptr_ + SANITIZER_GUARDED_BY(error_message_lock_); static void (*error_report_callback_)(const char *); }; -InternalMmapVector<char> *ScopedReport::error_message_ptr_; Mutex ScopedReport::error_message_lock_; +InternalScopedString *ScopedReport::error_message_ptr_; void (*ScopedReport::error_report_callback_)(const char *); // If there is an active ScopedReport, append to its error message. @@ -111,29 +109,45 @@ static void MaybePrintAndroidHelpUrl() { #endif } +namespace { // A RAII object that holds a copy of the current thread stack ring buffer. // The actual stack buffer may change while we are iterating over it (for // example, Printf may call syslog() which can itself be built with hwasan). class SavedStackAllocations { public: - SavedStackAllocations(StackAllocationsRingBuffer *rb) { + SavedStackAllocations() = default; + + explicit SavedStackAllocations(Thread *t) { CopyFrom(t); } + + void CopyFrom(Thread *t) { + StackAllocationsRingBuffer *rb = t->stack_allocations(); uptr size = rb->size() * sizeof(uptr); void *storage = MmapAlignedOrDieOnFatalError(size, size * 2, "saved stack allocations"); new (&rb_) StackAllocationsRingBuffer(*rb, storage); + thread_id_ = t->unique_id(); } ~SavedStackAllocations() { - StackAllocationsRingBuffer *rb = get(); - UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr)); + if (rb_) { + StackAllocationsRingBuffer *rb = get(); + UnmapOrDie(rb->StartOfStorage(), rb->size() * sizeof(uptr)); + } + } + + const StackAllocationsRingBuffer *get() const { + return (const StackAllocationsRingBuffer *)&rb_; } StackAllocationsRingBuffer *get() { return (StackAllocationsRingBuffer *)&rb_; } + u32 thread_id() const { return thread_id_; } + private: - uptr rb_; + uptr rb_ = 0; + u32 thread_id_; }; class Decorator: public __sanitizer::SanitizerCommonDecorator { @@ -146,6 +160,7 @@ class Decorator: public __sanitizer::SanitizerCommonDecorator { const char *Location() { return Green(); } const char *Thread() { return Green(); } }; +} // namespace static bool FindHeapAllocation(HeapAllocationsRingBuffer *rb, uptr tagged_addr, HeapAllocationRecord *har, uptr *ring_index, @@ -186,7 +201,7 @@ static bool FindHeapAllocation(HeapAllocationsRingBuffer *rb, uptr tagged_addr, return false; } -static void PrintStackAllocations(StackAllocationsRingBuffer *sa, +static void PrintStackAllocations(const StackAllocationsRingBuffer *sa, tag_t addr_tag, uptr untagged_addr) { uptr frames = Min((uptr)flags()->stack_history_size, sa->size()); bool found_local = false; @@ -242,12 +257,13 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa, break; uptr pc_mask = (1ULL << 48) - 1; uptr pc = record & pc_mask; - frame_desc.append(" record_addr:0x%zx record:0x%zx", - reinterpret_cast<uptr>(record_addr), record); + frame_desc.AppendF(" record_addr:0x%zx record:0x%zx", + reinterpret_cast<uptr>(record_addr), record); if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderFrame( + &frame_desc, " %F %L", 0, frame->info.address, &frame->info, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); frame->ClearAll(); } Printf("%s\n", frame_desc.data()); @@ -305,22 +321,342 @@ static uptr GetGlobalSizeFromDescriptor(uptr ptr) { return 0; } -static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate, - tag_t *left, tag_t *right) { - Decorator d; - uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); - HwasanChunkView chunk = FindHeapChunkByAddress(mem); +void ReportStats() {} + +constexpr uptr kDumpWidth = 16; +constexpr uptr kShadowLines = 17; +constexpr uptr kShadowDumpSize = kShadowLines * kDumpWidth; + +constexpr uptr kShortLines = 3; +constexpr uptr kShortDumpSize = kShortLines * kDumpWidth; +constexpr uptr kShortDumpOffset = (kShadowLines - kShortLines) / 2 * kDumpWidth; + +static uptr GetPrintTagStart(uptr addr) { + addr = MemToShadow(addr); + addr = RoundDownTo(addr, kDumpWidth); + addr -= kDumpWidth * (kShadowLines / 2); + return addr; +} + +template <typename PrintTag> +static void PrintTagInfoAroundAddr(uptr addr, uptr num_rows, + InternalScopedString &s, + PrintTag print_tag) { + uptr center_row_beg = RoundDownTo(addr, kDumpWidth); + uptr beg_row = center_row_beg - kDumpWidth * (num_rows / 2); + uptr end_row = center_row_beg + kDumpWidth * ((num_rows + 1) / 2); + for (uptr row = beg_row; row < end_row; row += kDumpWidth) { + s.Append(row == center_row_beg ? "=>" : " "); + s.AppendF("%p:", (void *)ShadowToMem(row)); + for (uptr i = 0; i < kDumpWidth; i++) { + s.Append(row + i == addr ? "[" : " "); + print_tag(s, row + i); + s.Append(row + i == addr ? "]" : " "); + } + s.AppendF("\n"); + } +} + +template <typename GetTag, typename GetShortTag> +static void PrintTagsAroundAddr(uptr addr, GetTag get_tag, + GetShortTag get_short_tag) { + InternalScopedString s; + addr = MemToShadow(addr); + s.AppendF( + "Memory tags around the buggy address (one tag corresponds to %zd " + "bytes):\n", + kShadowAlignment); + PrintTagInfoAroundAddr(addr, kShadowLines, s, + [&](InternalScopedString &s, uptr tag_addr) { + tag_t tag = get_tag(tag_addr); + s.AppendF("%02x", tag); + }); + + s.AppendF( + "Tags for short granules around the buggy address (one tag corresponds " + "to %zd bytes):\n", + kShadowAlignment); + PrintTagInfoAroundAddr(addr, kShortLines, s, + [&](InternalScopedString &s, uptr tag_addr) { + tag_t tag = get_tag(tag_addr); + if (tag >= 1 && tag <= kShadowAlignment) { + tag_t short_tag = get_short_tag(tag_addr); + s.AppendF("%02x", short_tag); + } else { + s.AppendF(".."); + } + }); + s.AppendF( + "See " + "https://clang.llvm.org/docs/" + "HardwareAssistedAddressSanitizerDesign.html#short-granules for a " + "description of short granule tags\n"); + Printf("%s", s.data()); +} + +static uptr GetTopPc(const StackTrace *stack) { + return stack->size ? StackTrace::GetPreviousInstructionPc(stack->trace[0]) + : 0; +} + +namespace { +class BaseReport { + public: + BaseReport(StackTrace *stack, bool fatal, uptr tagged_addr, uptr access_size) + : scoped_report(fatal), + stack(stack), + tagged_addr(tagged_addr), + access_size(access_size), + untagged_addr(UntagAddr(tagged_addr)), + ptr_tag(GetTagFromPointer(tagged_addr)), + mismatch_offset(FindMismatchOffset()), + heap(CopyHeapChunk()), + allocations(CopyAllocations()), + candidate(FindBufferOverflowCandidate()), + shadow(CopyShadow()) {} + + protected: + struct OverflowCandidate { + uptr untagged_addr = 0; + bool after = false; + bool is_close = false; + + struct { + uptr begin = 0; + uptr end = 0; + u32 thread_id = 0; + u32 stack_id = 0; + bool is_allocated = false; + } heap; + }; + + struct HeapAllocation { + HeapAllocationRecord har = {}; + uptr ring_index = 0; + uptr num_matching_addrs = 0; + uptr num_matching_addrs_4b = 0; + u32 free_thread_id = 0; + }; + + struct Allocations { + ArrayRef<SavedStackAllocations> stack; + ArrayRef<HeapAllocation> heap; + }; + + struct HeapChunk { + uptr begin = 0; + uptr size = 0; + u32 stack_id = 0; + bool from_small_heap = false; + bool is_allocated = false; + }; + + struct Shadow { + uptr addr = 0; + tag_t tags[kShadowDumpSize] = {}; + tag_t short_tags[kShortDumpSize] = {}; + }; + + sptr FindMismatchOffset() const; + Shadow CopyShadow() const; + tag_t GetTagCopy(uptr addr) const; + tag_t GetShortTagCopy(uptr addr) const; + HeapChunk CopyHeapChunk() const; + Allocations CopyAllocations(); + OverflowCandidate FindBufferOverflowCandidate() const; + void PrintAddressDescription() const; + void PrintHeapOrGlobalCandidate() const; + void PrintTags(uptr addr) const; + + SavedStackAllocations stack_allocations_storage[16]; + HeapAllocation heap_allocations_storage[256]; + + const ScopedReport scoped_report; + const StackTrace *stack = nullptr; + const uptr tagged_addr = 0; + const uptr access_size = 0; + const uptr untagged_addr = 0; + const tag_t ptr_tag = 0; + const sptr mismatch_offset = 0; + + const HeapChunk heap; + const Allocations allocations; + const OverflowCandidate candidate; + + const Shadow shadow; +}; + +sptr BaseReport::FindMismatchOffset() const { + if (!access_size) + return 0; + sptr offset = + __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); + CHECK_GE(offset, 0); + CHECK_LT(offset, static_cast<sptr>(access_size)); + tag_t *tag_ptr = + reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); + tag_t mem_tag = *tag_ptr; + + if (mem_tag && mem_tag < kShadowAlignment) { + tag_t *granule_ptr = reinterpret_cast<tag_t *>((untagged_addr + offset) & + ~(kShadowAlignment - 1)); + // If offset is 0, (untagged_addr + offset) is not aligned to granules. + // This is the offset of the leftmost accessed byte within the bad granule. + u8 in_granule_offset = (untagged_addr + offset) & (kShadowAlignment - 1); + tag_t short_tag = granule_ptr[kShadowAlignment - 1]; + // The first mismatch was a short granule that matched the ptr_tag. + if (short_tag == ptr_tag) { + // If the access starts after the end of the short granule, then the first + // bad byte is the first byte of the access; otherwise it is the first + // byte past the end of the short granule + if (mem_tag > in_granule_offset) { + offset += mem_tag - in_granule_offset; + } + } + } + return offset; +} + +BaseReport::Shadow BaseReport::CopyShadow() const { + Shadow result; + if (!MemIsApp(untagged_addr)) + return result; + + result.addr = GetPrintTagStart(untagged_addr + mismatch_offset); + uptr tag_addr = result.addr; + uptr short_end = kShortDumpOffset + ARRAY_SIZE(shadow.short_tags); + for (uptr i = 0; i < ARRAY_SIZE(result.tags); ++i, ++tag_addr) { + if (!MemIsShadow(tag_addr)) + continue; + result.tags[i] = *reinterpret_cast<tag_t *>(tag_addr); + if (i < kShortDumpOffset || i >= short_end) + continue; + uptr granule_addr = ShadowToMem(tag_addr); + if (1 <= result.tags[i] && result.tags[i] <= kShadowAlignment && + IsAccessibleMemoryRange(granule_addr, kShadowAlignment)) { + result.short_tags[i - kShortDumpOffset] = + *reinterpret_cast<tag_t *>(granule_addr + kShadowAlignment - 1); + } + } + return result; +} + +tag_t BaseReport::GetTagCopy(uptr addr) const { + CHECK_GE(addr, shadow.addr); + uptr idx = addr - shadow.addr; + CHECK_LT(idx, ARRAY_SIZE(shadow.tags)); + return shadow.tags[idx]; +} + +tag_t BaseReport::GetShortTagCopy(uptr addr) const { + CHECK_GE(addr, shadow.addr + kShortDumpOffset); + uptr idx = addr - shadow.addr - kShortDumpOffset; + CHECK_LT(idx, ARRAY_SIZE(shadow.short_tags)); + return shadow.short_tags[idx]; +} + +BaseReport::HeapChunk BaseReport::CopyHeapChunk() const { + HeapChunk result = {}; + if (MemIsShadow(untagged_addr)) + return result; + HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); + result.begin = chunk.Beg(); + if (result.begin) { + result.size = chunk.ActualSize(); + result.from_small_heap = chunk.FromSmallHeap(); + result.is_allocated = chunk.IsAllocated(); + result.stack_id = chunk.GetAllocStackId(); + } + return result; +} + +BaseReport::Allocations BaseReport::CopyAllocations() { + if (MemIsShadow(untagged_addr)) + return {}; + uptr stack_allocations_count = 0; + uptr heap_allocations_count = 0; + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + if (stack_allocations_count < ARRAY_SIZE(stack_allocations_storage) && + t->AddrIsInStack(untagged_addr)) { + stack_allocations_storage[stack_allocations_count++].CopyFrom(t); + } + + if (heap_allocations_count < ARRAY_SIZE(heap_allocations_storage)) { + // Scan all threads' ring buffers to find if it's a heap-use-after-free. + HeapAllocationRecord har; + uptr ring_index, num_matching_addrs, num_matching_addrs_4b; + if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har, + &ring_index, &num_matching_addrs, + &num_matching_addrs_4b)) { + auto &ha = heap_allocations_storage[heap_allocations_count++]; + ha.har = har; + ha.ring_index = ring_index; + ha.num_matching_addrs = num_matching_addrs; + ha.num_matching_addrs_4b = num_matching_addrs_4b; + ha.free_thread_id = t->unique_id(); + } + } + }); + + return {{stack_allocations_storage, stack_allocations_count}, + {heap_allocations_storage, heap_allocations_count}}; +} + +BaseReport::OverflowCandidate BaseReport::FindBufferOverflowCandidate() const { + OverflowCandidate result = {}; + if (MemIsShadow(untagged_addr)) + return result; + // Check if this looks like a heap buffer overflow by scanning + // the shadow left and right and looking for the first adjacent + // object with a different memory tag. If that tag matches ptr_tag, + // check the allocator if it has a live chunk there. + tag_t *tag_ptr = reinterpret_cast<tag_t *>(MemToShadow(untagged_addr)); + tag_t *candidate_tag_ptr = nullptr, *left = tag_ptr, *right = tag_ptr; + uptr candidate_distance = 0; + for (; candidate_distance < 1000; candidate_distance++) { + if (MemIsShadow(reinterpret_cast<uptr>(left)) && TagsEqual(ptr_tag, left)) { + candidate_tag_ptr = left; + break; + } + --left; + if (MemIsShadow(reinterpret_cast<uptr>(right)) && + TagsEqual(ptr_tag, right)) { + candidate_tag_ptr = right; + break; + } + ++right; + } + + constexpr auto kCloseCandidateDistance = 1; + result.is_close = candidate_distance <= kCloseCandidateDistance; + + result.after = candidate_tag_ptr == left; + result.untagged_addr = ShadowToMem(reinterpret_cast<uptr>(candidate_tag_ptr)); + HwasanChunkView chunk = FindHeapChunkByAddress(result.untagged_addr); if (chunk.IsAllocated()) { + result.heap.is_allocated = true; + result.heap.begin = chunk.Beg(); + result.heap.end = chunk.End(); + result.heap.thread_id = chunk.GetAllocThreadId(); + result.heap.stack_id = chunk.GetAllocStackId(); + } + return result; +} + +void BaseReport::PrintHeapOrGlobalCandidate() const { + Decorator d; + if (candidate.heap.is_allocated) { uptr offset; const char *whence; - if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) { - offset = untagged_addr - chunk.Beg(); + if (candidate.heap.begin <= untagged_addr && + untagged_addr < candidate.heap.end) { + offset = untagged_addr - candidate.heap.begin; whence = "inside"; - } else if (candidate == left) { - offset = untagged_addr - chunk.End(); + } else if (candidate.after) { + offset = untagged_addr - candidate.heap.end; whence = "after"; } else { - offset = chunk.Beg() - untagged_addr; + offset = candidate.heap.begin - untagged_addr; whence = "before"; } Printf("%s", d.Error()); @@ -328,12 +664,13 @@ static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate, Printf("%s", d.Default()); Printf("%s", d.Location()); Printf("%p is located %zd bytes %s a %zd-byte region [%p,%p)\n", - untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(), - chunk.End()); + untagged_addr, offset, whence, + candidate.heap.end - candidate.heap.begin, candidate.heap.begin, + candidate.heap.end); Printf("%s", d.Allocation()); - Printf("allocated by thread T%u here:\n", chunk.GetAllocThreadId()); + Printf("allocated by thread T%u here:\n", candidate.heap.thread_id); Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + GetStackTraceFromId(candidate.heap.stack_id).Print(); return; } // Check whether the address points into a loaded library. If so, this is @@ -341,47 +678,45 @@ static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate, const char *module_name; uptr module_address; Symbolizer *sym = Symbolizer::GetOrInit(); - if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) { + if (sym->GetModuleNameAndOffsetForPC(candidate.untagged_addr, &module_name, + &module_address)) { Printf("%s", d.Error()); Printf("\nCause: global-overflow\n"); Printf("%s", d.Default()); DataInfo info; Printf("%s", d.Location()); - if (sym->SymbolizeData(mem, &info) && info.start) { + if (sym->SymbolizeData(candidate.untagged_addr, &info) && info.start) { Printf( "%p is located %zd bytes %s a %zd-byte global variable " "%s [%p,%p) in %s\n", untagged_addr, - candidate == left ? untagged_addr - (info.start + info.size) - : info.start - untagged_addr, - candidate == left ? "after" : "before", info.size, info.name, + candidate.after ? untagged_addr - (info.start + info.size) + : info.start - untagged_addr, + candidate.after ? "after" : "before", info.size, info.name, info.start, info.start + info.size, module_name); } else { - uptr size = GetGlobalSizeFromDescriptor(mem); + uptr size = GetGlobalSizeFromDescriptor(candidate.untagged_addr); if (size == 0) // We couldn't find the size of the global from the descriptors. Printf( "%p is located %s a global variable in " "\n #0 0x%x (%s+0x%x)\n", - untagged_addr, candidate == left ? "after" : "before", mem, - module_name, module_address); + untagged_addr, candidate.after ? "after" : "before", + candidate.untagged_addr, module_name, module_address); else Printf( "%p is located %s a %zd-byte global variable in " "\n #0 0x%x (%s+0x%x)\n", - untagged_addr, candidate == left ? "after" : "before", size, mem, - module_name, module_address); + untagged_addr, candidate.after ? "after" : "before", size, + candidate.untagged_addr, module_name, module_address); } Printf("%s", d.Default()); } } -void PrintAddressDescription( - uptr tagged_addr, uptr access_size, - StackAllocationsRingBuffer *current_stack_allocations) { +void BaseReport::PrintAddressDescription() const { Decorator d; int num_descriptions_printed = 0; - uptr untagged_addr = UntagAddr(tagged_addr); if (MemIsShadow(untagged_addr)) { Printf("%s%p is HWAsan shadow memory.\n%s", d.Location(), untagged_addr, @@ -390,113 +725,80 @@ void PrintAddressDescription( } // Print some very basic information about the address, if it's a heap. - HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); - if (uptr beg = chunk.Beg()) { - uptr size = chunk.ActualSize(); - Printf("%s[%p,%p) is a %s %s heap chunk; " - "size: %zd offset: %zd\n%s", - d.Location(), - beg, beg + size, - chunk.FromSmallHeap() ? "small" : "large", - chunk.IsAllocated() ? "allocated" : "unallocated", - size, untagged_addr - beg, - d.Default()); + if (heap.begin) { + Printf( + "%s[%p,%p) is a %s %s heap chunk; " + "size: %zd offset: %zd\n%s", + d.Location(), heap.begin, heap.begin + heap.size, + heap.from_small_heap ? "small" : "large", + heap.is_allocated ? "allocated" : "unallocated", heap.size, + untagged_addr - heap.begin, d.Default()); } - tag_t addr_tag = GetTagFromPointer(tagged_addr); + auto announce_by_id = [](u32 thread_id) { + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + if (thread_id == t->unique_id()) + t->Announce(); + }); + }; - bool on_stack = false; // Check stack first. If the address is on the stack of a live thread, we // know it cannot be a heap / global overflow. - hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { - if (t->AddrIsInStack(untagged_addr)) { - on_stack = true; - // TODO(fmayer): figure out how to distinguish use-after-return and - // stack-buffer-overflow. - Printf("%s", d.Error()); - Printf("\nCause: stack tag-mismatch\n"); - Printf("%s", d.Location()); - Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, - t->unique_id()); - Printf("%s", d.Default()); - t->Announce(); - - auto *sa = (t == GetCurrentThread() && current_stack_allocations) - ? current_stack_allocations - : t->stack_allocations(); - PrintStackAllocations(sa, addr_tag, untagged_addr); - num_descriptions_printed++; - } - }); + for (const auto &sa : allocations.stack) { + // TODO(fmayer): figure out how to distinguish use-after-return and + // stack-buffer-overflow. + Printf("%s", d.Error()); + Printf("\nCause: stack tag-mismatch\n"); + Printf("%s", d.Location()); + Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, + sa.thread_id()); + Printf("%s", d.Default()); + announce_by_id(sa.thread_id()); + PrintStackAllocations(sa.get(), ptr_tag, untagged_addr); + num_descriptions_printed++; + } - // Check if this looks like a heap buffer overflow by scanning - // the shadow left and right and looking for the first adjacent - // object with a different memory tag. If that tag matches addr_tag, - // check the allocator if it has a live chunk there. - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); - tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr; - uptr candidate_distance = 0; - for (; candidate_distance < 1000; candidate_distance++) { - if (MemIsShadow(reinterpret_cast<uptr>(left)) && - TagsEqual(addr_tag, left)) { - candidate = left; - break; - } - --left; - if (MemIsShadow(reinterpret_cast<uptr>(right)) && - TagsEqual(addr_tag, right)) { - candidate = right; - break; - } - ++right; + if (allocations.stack.empty() && candidate.untagged_addr && + candidate.is_close) { + PrintHeapOrGlobalCandidate(); + num_descriptions_printed++; } - constexpr auto kCloseCandidateDistance = 1; + for (const auto &ha : allocations.heap) { + const HeapAllocationRecord har = ha.har; - if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) { - ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + Printf("%s", d.Error()); + Printf("\nCause: use-after-free\n"); + Printf("%s", d.Location()); + Printf("%p is located %zd bytes inside a %zd-byte region [%p,%p)\n", + untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), + har.requested_size, UntagAddr(har.tagged_addr), + UntagAddr(har.tagged_addr) + har.requested_size); + Printf("%s", d.Allocation()); + Printf("freed by thread T%u here:\n", ha.free_thread_id); + Printf("%s", d.Default()); + GetStackTraceFromId(har.free_context_id).Print(); + + Printf("%s", d.Allocation()); + Printf("previously allocated by thread T%u here:\n", har.alloc_thread_id); + Printf("%s", d.Default()); + GetStackTraceFromId(har.alloc_context_id).Print(); + + // Print a developer note: the index of this heap object + // in the thread's deallocation ring buffer. + Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", ha.ring_index + 1, + flags()->heap_history_size); + Printf("hwasan_dev_note_num_matching_addrs: %zd\n", ha.num_matching_addrs); + Printf("hwasan_dev_note_num_matching_addrs_4b: %zd\n", + ha.num_matching_addrs_4b); + + announce_by_id(ha.free_thread_id); + // TODO: announce_by_id(har.alloc_thread_id); num_descriptions_printed++; } - hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { - // Scan all threads' ring buffers to find if it's a heap-use-after-free. - HeapAllocationRecord har; - uptr ring_index, num_matching_addrs, num_matching_addrs_4b; - if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har, - &ring_index, &num_matching_addrs, - &num_matching_addrs_4b)) { - Printf("%s", d.Error()); - Printf("\nCause: use-after-free\n"); - Printf("%s", d.Location()); - Printf("%p is located %zd bytes inside a %zd-byte region [%p,%p)\n", - untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), - har.requested_size, UntagAddr(har.tagged_addr), - UntagAddr(har.tagged_addr) + har.requested_size); - Printf("%s", d.Allocation()); - Printf("freed by thread T%u here:\n", t->unique_id()); - Printf("%s", d.Default()); - GetStackTraceFromId(har.free_context_id).Print(); - - Printf("%s", d.Allocation()); - Printf("previously allocated by thread T%u here:\n", har.alloc_thread_id); - Printf("%s", d.Default()); - GetStackTraceFromId(har.alloc_context_id).Print(); - - // Print a developer note: the index of this heap object - // in the thread's deallocation ring buffer. - Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", ring_index + 1, - flags()->heap_history_size); - Printf("hwasan_dev_note_num_matching_addrs: %zd\n", num_matching_addrs); - Printf("hwasan_dev_note_num_matching_addrs_4b: %zd\n", - num_matching_addrs_4b); - - t->Announce(); - num_descriptions_printed++; - } - }); - - if (candidate && num_descriptions_printed == 0) { - ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + if (candidate.untagged_addr && num_descriptions_printed == 0) { + PrintHeapOrGlobalCandidate(); num_descriptions_printed++; } @@ -515,77 +817,24 @@ void PrintAddressDescription( } } -void ReportStats() {} - -static void PrintTagInfoAroundAddr(tag_t *tag_ptr, uptr num_rows, - void (*print_tag)(InternalScopedString &s, - tag_t *tag)) { - const uptr row_len = 16; // better be power of two. - tag_t *center_row_beg = reinterpret_cast<tag_t *>( - RoundDownTo(reinterpret_cast<uptr>(tag_ptr), row_len)); - tag_t *beg_row = center_row_beg - row_len * (num_rows / 2); - tag_t *end_row = center_row_beg + row_len * ((num_rows + 1) / 2); - InternalScopedString s; - for (tag_t *row = beg_row; row < end_row; row += row_len) { - s.append("%s", row == center_row_beg ? "=>" : " "); - s.append("%p:", (void *)ShadowToMem(reinterpret_cast<uptr>(row))); - for (uptr i = 0; i < row_len; i++) { - s.append("%s", row + i == tag_ptr ? "[" : " "); - print_tag(s, &row[i]); - s.append("%s", row + i == tag_ptr ? "]" : " "); - } - s.append("\n"); +void BaseReport::PrintTags(uptr addr) const { + if (shadow.addr) { + PrintTagsAroundAddr( + addr, [&](uptr addr) { return GetTagCopy(addr); }, + [&](uptr addr) { return GetShortTagCopy(addr); }); } - Printf("%s", s.data()); } -static void PrintTagsAroundAddr(tag_t *tag_ptr) { - Printf( - "Memory tags around the buggy address (one tag corresponds to %zd " - "bytes):\n", kShadowAlignment); - PrintTagInfoAroundAddr(tag_ptr, 17, [](InternalScopedString &s, tag_t *tag) { - s.append("%02x", *tag); - }); - - Printf( - "Tags for short granules around the buggy address (one tag corresponds " - "to %zd bytes):\n", - kShadowAlignment); - PrintTagInfoAroundAddr(tag_ptr, 3, [](InternalScopedString &s, tag_t *tag) { - if (*tag >= 1 && *tag <= kShadowAlignment) { - uptr granule_addr = ShadowToMem(reinterpret_cast<uptr>(tag)); - s.append("%02x", - *reinterpret_cast<u8 *>(granule_addr + kShadowAlignment - 1)); - } else { - s.append(".."); - } - }); - Printf( - "See " - "https://clang.llvm.org/docs/" - "HardwareAssistedAddressSanitizerDesign.html#short-granules for a " - "description of short granule tags\n"); -} +class InvalidFreeReport : public BaseReport { + public: + InvalidFreeReport(StackTrace *stack, uptr tagged_addr) + : BaseReport(stack, flags()->halt_on_error, tagged_addr, 0) {} + ~InvalidFreeReport(); -uptr GetTopPc(StackTrace *stack) { - return stack->size ? StackTrace::GetPreviousInstructionPc(stack->trace[0]) - : 0; -} + private: +}; -void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { - ScopedReport R(flags()->halt_on_error); - - uptr untagged_addr = UntagAddr(tagged_addr); - tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = nullptr; - tag_t mem_tag = 0; - if (MemIsApp(untagged_addr)) { - tag_ptr = reinterpret_cast<tag_t *>(MemToShadow(untagged_addr)); - if (MemIsShadow(reinterpret_cast<uptr>(tag_ptr))) - mem_tag = *tag_ptr; - else - tag_ptr = nullptr; - } +InvalidFreeReport::~InvalidFreeReport() { Decorator d; Printf("%s", d.Error()); uptr pc = GetTopPc(stack); @@ -599,36 +848,49 @@ void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { SanitizerToolName, bug_type, untagged_addr, pc); } Printf("%s", d.Access()); - if (tag_ptr) - Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, mem_tag); + if (shadow.addr) { + Printf("tags: %02x/%02x (ptr/mem)\n", ptr_tag, + GetTagCopy(MemToShadow(untagged_addr))); + } Printf("%s", d.Default()); stack->Print(); - PrintAddressDescription(tagged_addr, 0, nullptr); - - if (tag_ptr) - PrintTagsAroundAddr(tag_ptr); - + PrintAddressDescription(); + PrintTags(untagged_addr); MaybePrintAndroidHelpUrl(); ReportErrorSummary(bug_type, stack); } -void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, - const u8 *expected) { - uptr tail_size = kShadowAlignment - (orig_size % kShadowAlignment); - u8 actual_expected[kShadowAlignment]; - internal_memcpy(actual_expected, expected, tail_size); - tag_t ptr_tag = GetTagFromPointer(tagged_addr); - // Short granule is stashed in the last byte of the magic string. To avoid - // confusion, make the expected magic string contain the short granule tag. - if (orig_size % kShadowAlignment != 0) { - actual_expected[tail_size - 1] = ptr_tag; +class TailOverwrittenReport : public BaseReport { + public: + explicit TailOverwrittenReport(StackTrace *stack, uptr tagged_addr, + uptr orig_size, const u8 *expected) + : BaseReport(stack, flags()->halt_on_error, tagged_addr, 0), + orig_size(orig_size), + tail_size(kShadowAlignment - (orig_size % kShadowAlignment)) { + CHECK_GT(tail_size, 0U); + CHECK_LT(tail_size, kShadowAlignment); + internal_memcpy(tail_copy, + reinterpret_cast<u8 *>(untagged_addr + orig_size), + tail_size); + internal_memcpy(actual_expected, expected, tail_size); + // Short granule is stashed in the last byte of the magic string. To avoid + // confusion, make the expected magic string contain the short granule tag. + if (orig_size % kShadowAlignment != 0) + actual_expected[tail_size - 1] = ptr_tag; } + ~TailOverwrittenReport(); + + private: + const uptr orig_size = 0; + const uptr tail_size = 0; + u8 actual_expected[kShadowAlignment] = {}; + u8 tail_copy[kShadowAlignment] = {}; +}; - ScopedReport R(flags()->halt_on_error); +TailOverwrittenReport::~TailOverwrittenReport() { Decorator d; - uptr untagged_addr = UntagAddr(tagged_addr); Printf("%s", d.Error()); const char *bug_type = "allocation-tail-overwritten"; Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName, @@ -641,61 +903,62 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, Printf("deallocated here:\n"); Printf("%s", d.Default()); stack->Print(); - HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); - if (chunk.Beg()) { + if (heap.begin) { Printf("%s", d.Allocation()); Printf("allocated here:\n"); Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + GetStackTraceFromId(heap.stack_id).Print(); } InternalScopedString s; - CHECK_GT(tail_size, 0U); - CHECK_LT(tail_size, kShadowAlignment); - u8 *tail = reinterpret_cast<u8*>(untagged_addr + orig_size); - s.append("Tail contains: "); - for (uptr i = 0; i < kShadowAlignment - tail_size; i++) - s.append(".. "); + u8 *tail = tail_copy; + s.AppendF("Tail contains: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) s.AppendF(".. "); + for (uptr i = 0; i < tail_size; i++) s.AppendF("%02x ", tail[i]); + s.AppendF("\n"); + s.AppendF("Expected: "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) s.AppendF(".. "); + for (uptr i = 0; i < tail_size; i++) s.AppendF("%02x ", actual_expected[i]); + s.AppendF("\n"); + s.AppendF(" "); + for (uptr i = 0; i < kShadowAlignment - tail_size; i++) s.AppendF(" "); for (uptr i = 0; i < tail_size; i++) - s.append("%02x ", tail[i]); - s.append("\n"); - s.append("Expected: "); - for (uptr i = 0; i < kShadowAlignment - tail_size; i++) - s.append(".. "); - for (uptr i = 0; i < tail_size; i++) s.append("%02x ", actual_expected[i]); - s.append("\n"); - s.append(" "); - for (uptr i = 0; i < kShadowAlignment - tail_size; i++) - s.append(" "); - for (uptr i = 0; i < tail_size; i++) - s.append("%s ", actual_expected[i] != tail[i] ? "^^" : " "); - - s.append("\nThis error occurs when a buffer overflow overwrites memory\n" - "after a heap object, but within the %zd-byte granule, e.g.\n" - " char *x = new char[20];\n" - " x[25] = 42;\n" - "%s does not detect such bugs in uninstrumented code at the time of write," - "\nbut can detect them at the time of free/delete.\n" - "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0\n", - kShadowAlignment, SanitizerToolName); + s.AppendF("%s ", actual_expected[i] != tail[i] ? "^^" : " "); + + s.AppendF( + "\nThis error occurs when a buffer overflow overwrites memory\n" + "after a heap object, but within the %zd-byte granule, e.g.\n" + " char *x = new char[20];\n" + " x[25] = 42;\n" + "%s does not detect such bugs in uninstrumented code at the time of " + "write," + "\nbut can detect them at the time of free/delete.\n" + "To disable this feature set HWASAN_OPTIONS=free_checks_tail_magic=0\n", + kShadowAlignment, SanitizerToolName); Printf("%s", s.data()); GetCurrentThread()->Announce(); - - tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); - PrintTagsAroundAddr(tag_ptr); - + PrintTags(untagged_addr); MaybePrintAndroidHelpUrl(); ReportErrorSummary(bug_type, stack); } -void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, - bool is_store, bool fatal, uptr *registers_frame) { - ScopedReport R(fatal); - SavedStackAllocations current_stack_allocations( - GetCurrentThread()->stack_allocations()); +class TagMismatchReport : public BaseReport { + public: + explicit TagMismatchReport(StackTrace *stack, uptr tagged_addr, + uptr access_size, bool is_store, bool fatal, + uptr *registers_frame) + : BaseReport(stack, fatal, tagged_addr, access_size), + is_store(is_store), + registers_frame(registers_frame) {} + ~TagMismatchReport(); + + private: + const bool is_store; + const uptr *registers_frame; +}; +TagMismatchReport::~TagMismatchReport() { Decorator d; - uptr untagged_addr = UntagAddr(tagged_addr); // TODO: when possible, try to print heap-use-after-free, etc. const char *bug_type = "tag-mismatch"; uptr pc = GetTopPc(stack); @@ -705,31 +968,12 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, Thread *t = GetCurrentThread(); - sptr offset = - __hwasan_test_shadow(reinterpret_cast<void *>(tagged_addr), access_size); - CHECK(offset >= 0 && offset < static_cast<sptr>(access_size)); - tag_t ptr_tag = GetTagFromPointer(tagged_addr); - tag_t *tag_ptr = - reinterpret_cast<tag_t *>(MemToShadow(untagged_addr + offset)); - tag_t mem_tag = *tag_ptr; + tag_t mem_tag = GetTagCopy(MemToShadow(untagged_addr + mismatch_offset)); Printf("%s", d.Access()); if (mem_tag && mem_tag < kShadowAlignment) { - tag_t *granule_ptr = reinterpret_cast<tag_t *>((untagged_addr + offset) & - ~(kShadowAlignment - 1)); - // If offset is 0, (untagged_addr + offset) is not aligned to granules. - // This is the offset of the leftmost accessed byte within the bad granule. - u8 in_granule_offset = (untagged_addr + offset) & (kShadowAlignment - 1); - tag_t short_tag = granule_ptr[kShadowAlignment - 1]; - // The first mismatch was a short granule that matched the ptr_tag. - if (short_tag == ptr_tag) { - // If the access starts after the end of the short granule, then the first - // bad byte is the first byte of the access; otherwise it is the first - // byte past the end of the short granule - if (mem_tag > in_granule_offset) { - offset += mem_tag - in_granule_offset; - } - } + tag_t short_tag = + GetShortTagCopy(MemToShadow(untagged_addr + mismatch_offset)); Printf( "%s of size %zu at %p tags: %02x/%02x(%02x) (ptr/mem) in thread T%zd\n", is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, @@ -739,17 +983,16 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag, mem_tag, t->unique_id()); } - if (offset != 0) - Printf("Invalid access starting at offset %zu\n", offset); + if (mismatch_offset) + Printf("Invalid access starting at offset %zu\n", mismatch_offset); Printf("%s", d.Default()); stack->Print(); - PrintAddressDescription(tagged_addr, access_size, - current_stack_allocations.get()); + PrintAddressDescription(); t->Announce(); - PrintTagsAroundAddr(tag_ptr); + PrintTags(untagged_addr + mismatch_offset); if (registers_frame) ReportRegisters(registers_frame, pc); @@ -757,10 +1000,26 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, MaybePrintAndroidHelpUrl(); ReportErrorSummary(bug_type, stack); } +} // namespace + +void ReportInvalidFree(StackTrace *stack, uptr tagged_addr) { + InvalidFreeReport R(stack, tagged_addr); +} + +void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, + const u8 *expected) { + TailOverwrittenReport R(stack, tagged_addr, orig_size, expected); +} + +void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size, + bool is_store, bool fatal, uptr *registers_frame) { + TagMismatchReport R(stack, tagged_addr, access_size, is_store, fatal, + registers_frame); +} // See the frame breakdown defined in __hwasan_tag_mismatch (from // hwasan_tag_mismatch_{aarch64,riscv64}.S). -void ReportRegisters(uptr *frame, uptr pc) { +void ReportRegisters(const uptr *frame, uptr pc) { Printf("Registers where the failure occurred (pc %p):\n", pc); // We explicitly print a single line (4 registers/line) each iteration to @@ -772,7 +1031,8 @@ void ReportRegisters(uptr *frame, uptr pc) { frame[0], frame[1], frame[2], frame[3]); #elif SANITIZER_RISCV64 Printf(" sp %016llx x1 %016llx x2 %016llx x3 %016llx\n", - reinterpret_cast<u8 *>(frame) + 256, frame[1], frame[2], frame[3]); + reinterpret_cast<const u8 *>(frame) + 256, frame[1], frame[2], + frame[3]); #endif Printf(" x4 %016llx x5 %016llx x6 %016llx x7 %016llx\n", frame[4], frame[5], frame[6], frame[7]); @@ -790,7 +1050,7 @@ void ReportRegisters(uptr *frame, uptr pc) { // passes it to this function. #if defined(__aarch64__) Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28], - frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256); + frame[29], frame[30], reinterpret_cast<const u8 *>(frame) + 256); #elif SANITIZER_RISCV64 Printf(" x28 %016llx x29 %016llx x30 %016llx x31 %016llx\n", frame[28], frame[29], frame[30], frame[31]); diff --git a/libsanitizer/hwasan/hwasan_report.h b/libsanitizer/hwasan/hwasan_report.h index de86c38fc01..bb9492a18cf 100644 --- a/libsanitizer/hwasan/hwasan_report.h +++ b/libsanitizer/hwasan/hwasan_report.h @@ -26,7 +26,7 @@ void ReportTagMismatch(StackTrace *stack, uptr addr, uptr access_size, void ReportInvalidFree(StackTrace *stack, uptr addr); void ReportTailOverwritten(StackTrace *stack, uptr addr, uptr orig_size, const u8 *expected); -void ReportRegisters(uptr *registers_frame, uptr pc); +void ReportRegisters(const uptr *registers_frame, uptr pc); void ReportAtExitStatistics(); diff --git a/libsanitizer/hwasan/hwasan_setjmp_aarch64.S b/libsanitizer/hwasan/hwasan_setjmp_aarch64.S index 744748a5101..0c0abb6de86 100644 --- a/libsanitizer/hwasan/hwasan_setjmp_aarch64.S +++ b/libsanitizer/hwasan/hwasan_setjmp_aarch64.S @@ -31,33 +31,37 @@ .section .text .file "hwasan_setjmp_aarch64.S" -.global __interceptor_setjmp -ASM_TYPE_FUNCTION(__interceptor_setjmp) -__interceptor_setjmp: +.global ASM_WRAPPER_NAME(setjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(setjmp)) +ASM_WRAPPER_NAME(setjmp): CFI_STARTPROC BTI_C mov x1, #0 - b __interceptor_sigsetjmp + b ASM_WRAPPER_NAME(sigsetjmp) CFI_ENDPROC -ASM_SIZE(__interceptor_setjmp) +ASM_SIZE(ASM_WRAPPER_NAME(setjmp)) + +ASM_INTERCEPTOR_TRAMPOLINE(setjmp) #if SANITIZER_ANDROID // Bionic also defines a function `setjmp` that calls `sigsetjmp` saving the // current signal. -.global __interceptor_setjmp_bionic -ASM_TYPE_FUNCTION(__interceptor_setjmp_bionic) -__interceptor_setjmp_bionic: +.global ASM_WRAPPER_NAME(setjmp_bionic) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(setjmp_bionic)) +ASM_WRAPPER_NAME(setjmp_bionic): CFI_STARTPROC BTI_C mov x1, #1 - b __interceptor_sigsetjmp + b ASM_WRAPPER_NAME(sigsetjmp) CFI_ENDPROC -ASM_SIZE(__interceptor_setjmp_bionic) +ASM_SIZE(ASM_WRAPPER_NAME(setjmp_bionic)) + +ASM_INTERCEPTOR_TRAMPOLINE(setjmp_bionic) #endif -.global __interceptor_sigsetjmp -ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) -__interceptor_sigsetjmp: +.global ASM_WRAPPER_NAME(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(sigsetjmp)) +ASM_WRAPPER_NAME(sigsetjmp): CFI_STARTPROC BTI_C stp x19, x20, [x0, #0<<3] @@ -77,22 +81,19 @@ __interceptor_sigsetjmp: // This function is defined in hwasan_interceptors.cc b __sigjmp_save CFI_ENDPROC -ASM_SIZE(__interceptor_sigsetjmp) +ASM_SIZE(ASM_WRAPPER_NAME(sigsetjmp)) +ASM_INTERCEPTOR_TRAMPOLINE(sigsetjmp) -.macro WEAK_ALIAS first second - .weak \second - .equ \second\(), \first -.endm #if SANITIZER_ANDROID -WEAK_ALIAS __interceptor_sigsetjmp, sigsetjmp -WEAK_ALIAS __interceptor_setjmp_bionic, setjmp +ASM_TRAMPOLINE_ALIAS(sigsetjmp, sigsetjmp) +ASM_TRAMPOLINE_ALIAS(setjmp, setjmp_bionic) #else -WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp +ASM_TRAMPOLINE_ALIAS(__sigsetjmp, sigsetjmp) #endif -WEAK_ALIAS __interceptor_setjmp, _setjmp +ASM_TRAMPOLINE_ALIAS(_setjmp, setjmp) #endif // We do not need executable stack. diff --git a/libsanitizer/hwasan/hwasan_setjmp_riscv64.S b/libsanitizer/hwasan/hwasan_setjmp_riscv64.S index 43f9c3c26b4..c01f4e25e8a 100644 --- a/libsanitizer/hwasan/hwasan_setjmp_riscv64.S +++ b/libsanitizer/hwasan/hwasan_setjmp_riscv64.S @@ -31,18 +31,18 @@ .section .text .file "hwasan_setjmp_riscv64.S" -.global __interceptor_setjmp -ASM_TYPE_FUNCTION(__interceptor_setjmp) -__interceptor_setjmp: +.global ASM_WRAPPER_NAME(setjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(setjmp)) +ASM_WRAPPER_NAME(setjmp): CFI_STARTPROC addi x11, x0, 0 - tail __interceptor_sigsetjmp + tail ASM_WRAPPER_NAME(sigsetjmp) CFI_ENDPROC -ASM_SIZE(__interceptor_setjmp) +ASM_SIZE(ASM_WRAPPER_NAME(setjmp)) -.global __interceptor_sigsetjmp -ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) -__interceptor_sigsetjmp: +.global ASM_WRAPPER_NAME(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(sigsetjmp)) +ASM_WRAPPER_NAME(sigsetjmp): CFI_STARTPROC sd ra, 0<<3(x10) sd s0, 1<<3(x10) @@ -80,17 +80,12 @@ __interceptor_sigsetjmp: // This function is defined in hwasan_interceptors.cc tail __sigjmp_save CFI_ENDPROC -ASM_SIZE(__interceptor_sigsetjmp) +ASM_SIZE(ASM_WRAPPER_NAME(sigsetjmp)) - -.macro WEAK_ALIAS first second - .weak \second - .equ \second\(), \first -.endm - -WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp - -WEAK_ALIAS __interceptor_setjmp, _setjmp +ASM_INTERCEPTOR_TRAMPOLINE(sigsetjmp) +ASM_TRAMPOLINE_ALIAS(__sigsetjmp, sigsetjmp) +ASM_INTERCEPTOR_TRAMPOLINE(setjmp) +ASM_TRAMPOLINE_ALIAS(_setjmp, setjmp) #endif // We do not need executable stack. diff --git a/libsanitizer/hwasan/hwasan_setjmp_x86_64.S b/libsanitizer/hwasan/hwasan_setjmp_x86_64.S index a5a3858d94d..9804e8d7cec 100644 --- a/libsanitizer/hwasan/hwasan_setjmp_x86_64.S +++ b/libsanitizer/hwasan/hwasan_setjmp_x86_64.S @@ -31,19 +31,19 @@ .section .text .file "hwasan_setjmp_x86_64.S" -.global __interceptor_setjmp -ASM_TYPE_FUNCTION(__interceptor_setjmp) -__interceptor_setjmp: +.global ASM_WRAPPER_NAME(setjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(setjmp)) +ASM_WRAPPER_NAME(setjmp): CFI_STARTPROC _CET_ENDBR xorl %esi, %esi jmp .Linterceptor_sigsetjmp CFI_ENDPROC -ASM_SIZE(__interceptor_setjmp) +ASM_SIZE(ASM_WRAPPER_NAME(setjmp)) -.global __interceptor_sigsetjmp -ASM_TYPE_FUNCTION(__interceptor_sigsetjmp) -__interceptor_sigsetjmp: +.global ASM_WRAPPER_NAME(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_WRAPPER_NAME(sigsetjmp)) +ASM_WRAPPER_NAME(sigsetjmp): .Linterceptor_sigsetjmp: CFI_STARTPROC _CET_ENDBR @@ -67,16 +67,12 @@ __interceptor_sigsetjmp: jmp __sigjmp_save CFI_ENDPROC -ASM_SIZE(__interceptor_sigsetjmp) +ASM_SIZE(ASM_WRAPPER_NAME(sigsetjmp)) - -.macro WEAK_ALIAS first second - .weak \second - .equ \second\(), \first -.endm - -WEAK_ALIAS __interceptor_sigsetjmp, __sigsetjmp -WEAK_ALIAS __interceptor_setjmp, _setjmp +ASM_INTERCEPTOR_TRAMPOLINE(sigsetjmp) +ASM_TRAMPOLINE_ALIAS(__sigsetjmp, sigsetjmp) +ASM_INTERCEPTOR_TRAMPOLINE(setjmp) +ASM_TRAMPOLINE_ALIAS(_setjmp, setjmp) #endif // We do not need executable stack. diff --git a/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S b/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S index bcb0df42019..fd060c51cd8 100644 --- a/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S +++ b/libsanitizer/hwasan/hwasan_tag_mismatch_aarch64.S @@ -89,16 +89,16 @@ __hwasan_tag_mismatch: ubfx x16, x0, #4, #52 ldrb w16, [x9, x16] cmp w16, #0xf - b.hi __hwasan_tag_mismatch_v2 + b.hi mismatch cmp w16, w17 - b.lo __hwasan_tag_mismatch_v2 + b.lo mismatch // Load the real tag from the last byte of the granule and compare against // the pointer tag. orr x16, x0, #0xf ldrb w16, [x16] cmp x16, x0, lsr #56 - b.ne __hwasan_tag_mismatch_v2 + b.ne mismatch // Restore x0, x1 and sp to their values from before the __hwasan_tag_mismatch // call and resume execution. @@ -108,6 +108,8 @@ __hwasan_tag_mismatch: .global __hwasan_tag_mismatch_v2 .type __hwasan_tag_mismatch_v2, %function __hwasan_tag_mismatch_v2: +// Avoid using global label, to prevent "relocation out of range". +mismatch: CFI_STARTPROC BTI_J diff --git a/libsanitizer/hwasan/hwasan_thread.cpp b/libsanitizer/hwasan/hwasan_thread.cpp index 3375782ef29..ce36547580e 100644 --- a/libsanitizer/hwasan/hwasan_thread.cpp +++ b/libsanitizer/hwasan/hwasan_thread.cpp @@ -58,6 +58,16 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, #endif InitStackAndTls(state); dtls_ = DTLS_Get(); + AllocatorThreadStart(allocator_cache()); + + if (flags()->verbose_threads) { + if (IsMainThread()) { + Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", + sizeof(Thread), heap_allocations_->SizeInBytes(), + stack_allocations_->size() * sizeof(uptr)); + } + Print("Creating : "); + } } void Thread::InitStackRingBuffer(uptr stack_buffer_start, @@ -79,28 +89,23 @@ void Thread::InitStackRingBuffer(uptr stack_buffer_start, CHECK(MemIsApp(stack_bottom_)); CHECK(MemIsApp(stack_top_ - 1)); } - - if (flags()->verbose_threads) { - if (IsMainThread()) { - Printf("sizeof(Thread): %zd sizeof(HeapRB): %zd sizeof(StackRB): %zd\n", - sizeof(Thread), heap_allocations_->SizeInBytes(), - stack_allocations_->size() * sizeof(uptr)); - } - Print("Creating : "); - } } void Thread::ClearShadowForThreadStackAndTLS() { if (stack_top_ != stack_bottom_) - TagMemory(stack_bottom_, stack_top_ - stack_bottom_, 0); + TagMemory(UntagAddr(stack_bottom_), + UntagAddr(stack_top_) - UntagAddr(stack_bottom_), + GetTagFromPointer(stack_top_)); if (tls_begin_ != tls_end_) - TagMemory(tls_begin_, tls_end_ - tls_begin_, 0); + TagMemory(UntagAddr(tls_begin_), + UntagAddr(tls_end_) - UntagAddr(tls_begin_), + GetTagFromPointer(tls_begin_)); } void Thread::Destroy() { if (flags()->verbose_threads) Print("Destroying: "); - AllocatorSwallowThreadLocalCache(allocator_cache()); + AllocatorThreadFinish(allocator_cache()); ClearShadowForThreadStackAndTLS(); if (heap_allocations_) heap_allocations_->Delete(); @@ -173,9 +178,15 @@ static __hwasan::Thread *GetThreadByOsIDLocked(tid_t os_id) { [os_id](__hwasan::Thread *t) { return t->os_id() == os_id; }); } -void LockThreadRegistry() { __hwasan::hwasanThreadList().Lock(); } +void LockThreads() { + __hwasan::hwasanThreadList().Lock(); + __hwasan::hwasanThreadArgRetval().Lock(); +} -void UnlockThreadRegistry() { __hwasan::hwasanThreadList().Unlock(); } +void UnlockThreads() { + __hwasan::hwasanThreadArgRetval().Unlock(); + __hwasan::hwasanThreadList().Unlock(); +} void EnsureMainThreadIDIsCorrect() { __hwasan::EnsureMainThreadIDIsCorrect(); } @@ -202,7 +213,10 @@ void GetThreadExtraStackRangesLocked(tid_t os_id, InternalMmapVector<Range> *ranges) {} void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) {} -void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) {} +void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) { + __hwasan::hwasanThreadArgRetval().GetAllPtrsLocked(ptrs); +} + void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) {} } // namespace __lsan diff --git a/libsanitizer/hwasan/hwasan_thread_list.cpp b/libsanitizer/hwasan/hwasan_thread_list.cpp index fa46e658b69..7df4dd3d785 100644 --- a/libsanitizer/hwasan/hwasan_thread_list.cpp +++ b/libsanitizer/hwasan/hwasan_thread_list.cpp @@ -1,15 +1,28 @@ #include "hwasan_thread_list.h" +#include "sanitizer_common/sanitizer_thread_arg_retval.h" + namespace __hwasan { -static ALIGNED(16) char thread_list_placeholder[sizeof(HwasanThreadList)]; + static HwasanThreadList *hwasan_thread_list; +static ThreadArgRetval *thread_data; HwasanThreadList &hwasanThreadList() { return *hwasan_thread_list; } +ThreadArgRetval &hwasanThreadArgRetval() { return *thread_data; } void InitThreadList(uptr storage, uptr size) { - CHECK(hwasan_thread_list == nullptr); + CHECK_EQ(hwasan_thread_list, nullptr); + + static ALIGNED(alignof( + HwasanThreadList)) char thread_list_placeholder[sizeof(HwasanThreadList)]; hwasan_thread_list = new (thread_list_placeholder) HwasanThreadList(storage, size); + + CHECK_EQ(thread_data, nullptr); + + static ALIGNED(alignof( + ThreadArgRetval)) char thread_data_placeholder[sizeof(ThreadArgRetval)]; + thread_data = new (thread_data_placeholder) ThreadArgRetval(); } -} // namespace __hwasan +} // namespace __hwasan diff --git a/libsanitizer/hwasan/hwasan_thread_list.h b/libsanitizer/hwasan/hwasan_thread_list.h index 97485b195b6..82f6c70a03f 100644 --- a/libsanitizer/hwasan/hwasan_thread_list.h +++ b/libsanitizer/hwasan/hwasan_thread_list.h @@ -47,8 +47,8 @@ #include "hwasan_allocator.h" #include "hwasan_flags.h" #include "hwasan_thread.h" - #include "sanitizer_common/sanitizer_placement_new.h" +#include "sanitizer_common/sanitizer_thread_arg_retval.h" namespace __hwasan { @@ -131,9 +131,9 @@ class SANITIZER_MUTEX HwasanThreadList { void ReleaseThread(Thread *t) SANITIZER_EXCLUDES(free_list_mutex_) { RemoveThreadStats(t); + RemoveThreadFromLiveList(t); t->Destroy(); DontNeedThread(t); - RemoveThreadFromLiveList(t); SpinMutexLock l(&free_list_mutex_); free_list_.push_back(t); } @@ -157,7 +157,7 @@ class SANITIZER_MUTEX HwasanThreadList { } template <class CB> - Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(stats_mutex_) { + Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(live_list_mutex_) { CheckLocked(); for (Thread *t : live_list_) if (cb(t)) @@ -199,7 +199,7 @@ class SANITIZER_MUTEX HwasanThreadList { CHECK(IsAligned(free_space_, align)); Thread *t = (Thread *)(free_space_ + ring_buffer_size_); free_space_ += thread_alloc_size_; - CHECK(free_space_ <= free_space_end_ && "out of thread memory"); + CHECK_LE(free_space_, free_space_end_); return t; } @@ -222,5 +222,6 @@ class SANITIZER_MUTEX HwasanThreadList { void InitThreadList(uptr storage, uptr size); HwasanThreadList &hwasanThreadList(); +ThreadArgRetval &hwasanThreadArgRetval(); } // namespace __hwasan diff --git a/libsanitizer/include/sanitizer/allocator_interface.h b/libsanitizer/include/sanitizer/allocator_interface.h index d0cfce79c1a..a792e9f0136 100644 --- a/libsanitizer/include/sanitizer/allocator_interface.h +++ b/libsanitizer/include/sanitizer/allocator_interface.h @@ -11,82 +11,89 @@ #ifndef SANITIZER_ALLOCATOR_INTERFACE_H #define SANITIZER_ALLOCATOR_INTERFACE_H +#include <sanitizer/common_interface_defs.h> #include <stddef.h> #ifdef __cplusplus extern "C" { #endif - /* Returns the estimated number of bytes that will be reserved by allocator - for request of "size" bytes. If allocator can't allocate that much - memory, returns the maximal possible allocation size, otherwise returns - "size". */ - size_t __sanitizer_get_estimated_allocated_size(size_t size); +/* Returns the estimated number of bytes that will be reserved by allocator + for request of "size" bytes. If allocator can't allocate that much + memory, returns the maximal possible allocation size, otherwise returns + "size". */ +size_t SANITIZER_CDECL __sanitizer_get_estimated_allocated_size(size_t size); - /* Returns true if p was returned by the allocator and - is not yet freed. */ - int __sanitizer_get_ownership(const volatile void *p); +/* Returns true if p was returned by the allocator and + is not yet freed. */ +int SANITIZER_CDECL __sanitizer_get_ownership(const volatile void *p); - /* If a pointer lies within an allocation, it will return the start address - of the allocation. Otherwise, it returns nullptr. */ - const void *__sanitizer_get_allocated_begin(const void *p); +/* If a pointer lies within an allocation, it will return the start address + of the allocation. Otherwise, it returns nullptr. */ +const void *SANITIZER_CDECL __sanitizer_get_allocated_begin(const void *p); - /* Returns the number of bytes reserved for the pointer p. - Requires (get_ownership(p) == true) or (p == 0). */ - size_t __sanitizer_get_allocated_size(const volatile void *p); +/* Returns the number of bytes reserved for the pointer p. + Requires (get_ownership(p) == true) or (p == 0). */ +size_t SANITIZER_CDECL __sanitizer_get_allocated_size(const volatile void *p); - /* Number of bytes, allocated and not yet freed by the application. */ - size_t __sanitizer_get_current_allocated_bytes(void); +/* Returns the number of bytes reserved for the pointer p. + Requires __sanitizer_get_allocated_begin(p) == p. */ +size_t SANITIZER_CDECL +__sanitizer_get_allocated_size_fast(const volatile void *p); - /* Number of bytes, mmaped by the allocator to fulfill allocation requests. - Generally, for request of X bytes, allocator can reserve and add to free - lists a large number of chunks of size X to use them for future requests. - All these chunks count toward the heap size. Currently, allocator never - releases memory to OS (instead, it just puts freed chunks to free - lists). */ - size_t __sanitizer_get_heap_size(void); +/* Number of bytes, allocated and not yet freed by the application. */ +size_t SANITIZER_CDECL __sanitizer_get_current_allocated_bytes(void); - /* Number of bytes, mmaped by the allocator, which can be used to fulfill - allocation requests. When a user program frees memory chunk, it can first - fall into quarantine and will count toward __sanitizer_get_free_bytes() - later. */ - size_t __sanitizer_get_free_bytes(void); +/* Number of bytes, mmaped by the allocator to fulfill allocation requests. + Generally, for request of X bytes, allocator can reserve and add to free + lists a large number of chunks of size X to use them for future requests. + All these chunks count toward the heap size. Currently, allocator never + releases memory to OS (instead, it just puts freed chunks to free + lists). */ +size_t SANITIZER_CDECL __sanitizer_get_heap_size(void); - /* Number of bytes in unmapped pages, that are released to OS. Currently, - always returns 0. */ - size_t __sanitizer_get_unmapped_bytes(void); +/* Number of bytes, mmaped by the allocator, which can be used to fulfill + allocation requests. When a user program frees memory chunk, it can first + fall into quarantine and will count toward __sanitizer_get_free_bytes() + later. */ +size_t SANITIZER_CDECL __sanitizer_get_free_bytes(void); - /* Malloc hooks that may be optionally provided by user. - __sanitizer_malloc_hook(ptr, size) is called immediately after - allocation of "size" bytes, which returned "ptr". - __sanitizer_free_hook(ptr) is called immediately before - deallocation of "ptr". */ - void __sanitizer_malloc_hook(const volatile void *ptr, size_t size); - void __sanitizer_free_hook(const volatile void *ptr); +/* Number of bytes in unmapped pages, that are released to OS. Currently, + always returns 0. */ +size_t SANITIZER_CDECL __sanitizer_get_unmapped_bytes(void); - /* Installs a pair of hooks for malloc/free. - Several (currently, 5) hook pairs may be installed, they are executed - in the order they were installed and after calling - __sanitizer_malloc_hook/__sanitizer_free_hook. - Unlike __sanitizer_malloc_hook/__sanitizer_free_hook these hooks can be - chained and do not rely on weak symbols working on the platform, but - require __sanitizer_install_malloc_and_free_hooks to be called at startup - and thus will not be called on malloc/free very early in the process. - Returns the number of hooks currently installed or 0 on failure. - Not thread-safe, should be called in the main thread before starting - other threads. - */ - int __sanitizer_install_malloc_and_free_hooks( - void (*malloc_hook)(const volatile void *, size_t), - void (*free_hook)(const volatile void *)); +/* Malloc hooks that may be optionally provided by user. + __sanitizer_malloc_hook(ptr, size) is called immediately after + allocation of "size" bytes, which returned "ptr". + __sanitizer_free_hook(ptr) is called immediately before + deallocation of "ptr". */ +void SANITIZER_CDECL __sanitizer_malloc_hook(const volatile void *ptr, + size_t size); +void SANITIZER_CDECL __sanitizer_free_hook(const volatile void *ptr); - /* Drains allocator quarantines (calling thread's and global ones), returns - freed memory back to OS and releases other non-essential internal allocator - resources in attempt to reduce process RSS. - Currently available with ASan only. - */ - void __sanitizer_purge_allocator(void); +/* Installs a pair of hooks for malloc/free. + Several (currently, 5) hook pairs may be installed, they are executed + in the order they were installed and after calling + __sanitizer_malloc_hook/__sanitizer_free_hook. + Unlike __sanitizer_malloc_hook/__sanitizer_free_hook these hooks can be + chained and do not rely on weak symbols working on the platform, but + require __sanitizer_install_malloc_and_free_hooks to be called at startup + and thus will not be called on malloc/free very early in the process. + Returns the number of hooks currently installed or 0 on failure. + Not thread-safe, should be called in the main thread before starting + other threads. +*/ +int SANITIZER_CDECL __sanitizer_install_malloc_and_free_hooks( + void(SANITIZER_CDECL *malloc_hook)(const volatile void *, size_t), + void(SANITIZER_CDECL *free_hook)(const volatile void *)); + +/* Drains allocator quarantines (calling thread's and global ones), returns + freed memory back to OS and releases other non-essential internal allocator + resources in attempt to reduce process RSS. + Currently available with ASan only. +*/ +void SANITIZER_CDECL __sanitizer_purge_allocator(void); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif #endif diff --git a/libsanitizer/include/sanitizer/asan_interface.h b/libsanitizer/include/sanitizer/asan_interface.h index 9bff21c117b..37b6d08f4db 100644 --- a/libsanitizer/include/sanitizer/asan_interface.h +++ b/libsanitizer/include/sanitizer/asan_interface.h @@ -31,7 +31,8 @@ extern "C" { /// /// \param addr Start of memory region. /// \param size Size of memory region. -void __asan_poison_memory_region(void const volatile *addr, size_t size); +void SANITIZER_CDECL __asan_poison_memory_region(void const volatile *addr, + size_t size); /// Marks a memory region (<c>[addr, addr+size)</c>) as addressable. /// @@ -45,10 +46,19 @@ void __asan_poison_memory_region(void const volatile *addr, size_t size); /// /// \param addr Start of memory region. /// \param size Size of memory region. -void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +void SANITIZER_CDECL __asan_unpoison_memory_region(void const volatile *addr, + size_t size); // Macros provided for convenience. -#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) +#ifdef __has_feature +#if __has_feature(address_sanitizer) +#define ASAN_DEFINE_REGION_MACROS +#endif +#elif defined(__SANITIZE_ADDRESS__) +#define ASAN_DEFINE_REGION_MACROS +#endif + +#ifdef ASAN_DEFINE_REGION_MACROS /// Marks a memory region as unaddressable. /// /// \note Macro provided for convenience; defined as a no-op if ASan is not @@ -56,7 +66,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); /// /// \param addr Start of memory region. /// \param size Size of memory region. -#define ASAN_POISON_MEMORY_REGION(addr, size) \ +#define ASAN_POISON_MEMORY_REGION(addr, size) \ __asan_poison_memory_region((addr), (size)) /// Marks a memory region as addressable. @@ -66,14 +76,13 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); /// /// \param addr Start of memory region. /// \param size Size of memory region. -#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ __asan_unpoison_memory_region((addr), (size)) #else -#define ASAN_POISON_MEMORY_REGION(addr, size) \ - ((void)(addr), (void)(size)) -#define ASAN_UNPOISON_MEMORY_REGION(addr, size) \ - ((void)(addr), (void)(size)) +#define ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size)) +#define ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size)) #endif +#undef ASAN_DEFINE_REGION_MACROS /// Checks if an address is poisoned. /// @@ -85,7 +94,7 @@ void __asan_unpoison_memory_region(void const volatile *addr, size_t size); /// /// \retval 1 Address is poisoned. /// \retval 0 Address is not poisoned. -int __asan_address_is_poisoned(void const volatile *addr); +int SANITIZER_CDECL __asan_address_is_poisoned(void const volatile *addr); /// Checks if a region is poisoned. /// @@ -95,14 +104,14 @@ int __asan_address_is_poisoned(void const volatile *addr); /// \param beg Start of memory region. /// \param size Start of memory region. /// \returns Address of first poisoned byte. -void *__asan_region_is_poisoned(void *beg, size_t size); +void *SANITIZER_CDECL __asan_region_is_poisoned(void *beg, size_t size); /// Describes an address (useful for calling from the debugger). /// /// Prints the description of <c><i>addr</i></c>. /// /// \param addr Address to describe. -void __asan_describe_address(void *addr); +void SANITIZER_CDECL __asan_describe_address(void *addr); /// Checks if an error has been or is being reported (useful for calling from /// the debugger to get information about an ASan error). @@ -111,7 +120,7 @@ void __asan_describe_address(void *addr); /// /// \returns 1 if an error has been (or is being) reported. Otherwise returns /// 0. -int __asan_report_present(void); +int SANITIZER_CDECL __asan_report_present(void); /// Gets the PC (program counter) register value of an ASan error (useful for /// calling from the debugger). @@ -120,7 +129,7 @@ int __asan_report_present(void); /// Otherwise returns 0. /// /// \returns PC value. -void *__asan_get_report_pc(void); +void *SANITIZER_CDECL __asan_get_report_pc(void); /// Gets the BP (base pointer) register value of an ASan error (useful for /// calling from the debugger). @@ -129,7 +138,7 @@ void *__asan_get_report_pc(void); /// Otherwise returns 0. /// /// \returns BP value. -void *__asan_get_report_bp(void); +void *SANITIZER_CDECL __asan_get_report_bp(void); /// Gets the SP (stack pointer) register value of an ASan error (useful for /// calling from the debugger). @@ -138,7 +147,7 @@ void *__asan_get_report_bp(void); /// Otherwise returns 0. /// /// \returns SP value. -void *__asan_get_report_sp(void); +void *SANITIZER_CDECL __asan_get_report_sp(void); /// Gets the address of the report buffer of an ASan error (useful for calling /// from the debugger). @@ -147,7 +156,7 @@ void *__asan_get_report_sp(void); /// reported. Otherwise returns 0. /// /// \returns Address of report buffer. -void *__asan_get_report_address(void); +void *SANITIZER_CDECL __asan_get_report_address(void); /// Gets access type of an ASan error (useful for calling from the debugger). /// @@ -155,7 +164,7 @@ void *__asan_get_report_address(void); /// reported. Otherwise returns 0. /// /// \returns Access type (0 = read, 1 = write). -int __asan_get_report_access_type(void); +int SANITIZER_CDECL __asan_get_report_access_type(void); /// Gets access size of an ASan error (useful for calling from the debugger). /// @@ -163,7 +172,7 @@ int __asan_get_report_access_type(void); /// returns 0. /// /// \returns Access size in bytes. -size_t __asan_get_report_access_size(void); +size_t SANITIZER_CDECL __asan_get_report_access_size(void); /// Gets the bug description of an ASan error (useful for calling from a /// debugger). @@ -171,7 +180,7 @@ size_t __asan_get_report_access_size(void); /// \returns Returns a bug description if an error has been (or is being) /// reported - for example, "heap-use-after-free". Otherwise returns an empty /// string. -const char *__asan_get_report_description(void); +const char *SANITIZER_CDECL __asan_get_report_description(void); /// Gets information about a pointer (useful for calling from the debugger). /// @@ -192,8 +201,10 @@ const char *__asan_get_report_description(void); /// \param[out] region_size Size of the region in bytes. /// /// \returns Returns the category of the given pointer as a constant string. -const char *__asan_locate_address(void *addr, char *name, size_t name_size, - void **region_address, size_t *region_size); +const char *SANITIZER_CDECL __asan_locate_address(void *addr, char *name, + size_t name_size, + void **region_address, + size_t *region_size); /// Gets the allocation stack trace and thread ID for a heap address (useful /// for calling from the debugger). @@ -207,8 +218,8 @@ const char *__asan_locate_address(void *addr, char *name, size_t name_size, /// \param[out] thread_id The thread ID of the address. /// /// \returns Returns the number of stored frames or 0 on error. -size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size, - int *thread_id); +size_t SANITIZER_CDECL __asan_get_alloc_stack(void *addr, void **trace, + size_t size, int *thread_id); /// Gets the free stack trace and thread ID for a heap address (useful for /// calling from the debugger). @@ -222,15 +233,16 @@ size_t __asan_get_alloc_stack(void *addr, void **trace, size_t size, /// \param[out] thread_id The thread ID of the address. /// /// \returns Returns the number of stored frames or 0 on error. -size_t __asan_get_free_stack(void *addr, void **trace, size_t size, - int *thread_id); +size_t SANITIZER_CDECL __asan_get_free_stack(void *addr, void **trace, + size_t size, int *thread_id); /// Gets the current shadow memory mapping (useful for calling from the /// debugger). /// /// \param[out] shadow_scale Shadow scale value. /// \param[out] shadow_offset Offset value. -void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset); +void SANITIZER_CDECL __asan_get_shadow_mapping(size_t *shadow_scale, + size_t *shadow_offset); /// This is an internal function that is called to report an error. However, /// it is still a part of the interface because you might want to set a @@ -242,29 +254,31 @@ void __asan_get_shadow_mapping(size_t *shadow_scale, size_t *shadow_offset); /// \param addr Address of the ASan error. /// \param is_write True if the error is a write error; false otherwise. /// \param access_size Size of the memory access of the ASan error. -void __asan_report_error(void *pc, void *bp, void *sp, - void *addr, int is_write, size_t access_size); +void SANITIZER_CDECL __asan_report_error(void *pc, void *bp, void *sp, + void *addr, int is_write, + size_t access_size); // Deprecated. Call __sanitizer_set_death_callback instead. -void __asan_set_death_callback(void (*callback)(void)); +void SANITIZER_CDECL __asan_set_death_callback(void (*callback)(void)); /// Sets the callback function to be called during ASan error reporting. /// /// The callback provides a string pointer to the report. /// /// \param callback User-provided function. -void __asan_set_error_report_callback(void (*callback)(const char *)); +void SANITIZER_CDECL +__asan_set_error_report_callback(void (*callback)(const char *)); /// User-provided callback on ASan errors. /// /// You can provide a function that would be called immediately when ASan /// detects an error. This is useful in cases when ASan detects an error but /// your program crashes before the ASan report is printed. -void __asan_on_error(void); +void SANITIZER_CDECL __asan_on_error(void); /// Prints accumulated statistics to <c>stderr</c> (useful for calling from the /// debugger). -void __asan_print_accumulated_stats(void); +void SANITIZER_CDECL __asan_print_accumulated_stats(void); /// User-provided default option settings. /// @@ -273,7 +287,7 @@ void __asan_print_accumulated_stats(void); /// <c>verbosity=1:halt_on_error=0</c>). /// /// \returns Default options string. -const char* __asan_default_options(void); +const char *SANITIZER_CDECL __asan_default_options(void); // The following two functions facilitate garbage collection in presence of // ASan's fake stack. @@ -285,7 +299,7 @@ const char* __asan_default_options(void); /// does not have a fake stack. /// /// \returns An opaque handler to the fake stack or NULL. -void *__asan_get_current_fake_stack(void); +void *SANITIZER_CDECL __asan_get_current_fake_stack(void); /// Checks if an address belongs to a given fake stack. /// @@ -305,22 +319,22 @@ void *__asan_get_current_fake_stack(void); /// \param[out] beg Beginning of fake frame. /// \param[out] end End of fake frame. /// \returns Stack address or NULL. -void *__asan_addr_is_in_fake_stack(void *fake_stack, void *addr, void **beg, - void **end); +void *SANITIZER_CDECL __asan_addr_is_in_fake_stack(void *fake_stack, void *addr, + void **beg, void **end); /// Performs shadow memory cleanup of the current thread's stack before a /// function marked with the <c>[[noreturn]]</c> attribute is called. /// /// To avoid false positives on the stack, must be called before no-return /// functions like <c>_exit()</c> and <c>execl()</c>. -void __asan_handle_no_return(void); +void SANITIZER_CDECL __asan_handle_no_return(void); /// Update allocation stack trace for the given allocation to the current stack /// trace. Returns 1 if successful, 0 if not. -int __asan_update_allocation_context(void* addr); +int SANITIZER_CDECL __asan_update_allocation_context(void *addr); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_ASAN_INTERFACE_H +#endif // SANITIZER_ASAN_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/common_interface_defs.h b/libsanitizer/include/sanitizer/common_interface_defs.h index 2f415bd9e85..56d9e008fa0 100644 --- a/libsanitizer/include/sanitizer/common_interface_defs.h +++ b/libsanitizer/include/sanitizer/common_interface_defs.h @@ -15,9 +15,12 @@ #include <stddef.h> #include <stdint.h> -// GCC does not understand __has_feature. -#if !defined(__has_feature) -#define __has_feature(x) 0 +// Windows allows a user to set their default calling convention, but we always +// use __cdecl +#ifdef _WIN32 +#define SANITIZER_CDECL __cdecl +#else +#define SANITIZER_CDECL #endif #ifdef __cplusplus @@ -39,71 +42,73 @@ typedef struct { } __sanitizer_sandbox_arguments; // Tell the tools to write their reports to "path.<pid>" instead of stderr. -void __sanitizer_set_report_path(const char *path); +void SANITIZER_CDECL __sanitizer_set_report_path(const char *path); // Tell the tools to write their reports to the provided file descriptor // (casted to void *). -void __sanitizer_set_report_fd(void *fd); +void SANITIZER_CDECL __sanitizer_set_report_fd(void *fd); // Get the current full report file path, if a path was specified by // an earlier call to __sanitizer_set_report_path. Returns null otherwise. -const char *__sanitizer_get_report_path(); +const char *SANITIZER_CDECL __sanitizer_get_report_path(); // Notify the tools that the sandbox is going to be turned on. The reserved // parameter will be used in the future to hold a structure with functions // that the tools may call to bypass the sandbox. -void __sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); +void SANITIZER_CDECL +__sanitizer_sandbox_on_notify(__sanitizer_sandbox_arguments *args); // This function is called by the tool when it has just finished reporting // an error. 'error_summary' is a one-line string that summarizes // the error message. This function can be overridden by the client. -void __sanitizer_report_error_summary(const char *error_summary); +void SANITIZER_CDECL +__sanitizer_report_error_summary(const char *error_summary); // Some of the sanitizers (for example ASan/TSan) could miss bugs that happen // in unaligned loads/stores. To find such bugs reliably, you need to replace // plain unaligned loads/stores with these calls. /// Loads a 16-bit unaligned value. -/// +// /// \param p Pointer to unaligned memory. /// /// \returns Loaded value. -uint16_t __sanitizer_unaligned_load16(const void *p); +uint16_t SANITIZER_CDECL __sanitizer_unaligned_load16(const void *p); /// Loads a 32-bit unaligned value. /// /// \param p Pointer to unaligned memory. /// /// \returns Loaded value. -uint32_t __sanitizer_unaligned_load32(const void *p); +uint32_t SANITIZER_CDECL __sanitizer_unaligned_load32(const void *p); /// Loads a 64-bit unaligned value. /// /// \param p Pointer to unaligned memory. /// /// \returns Loaded value. -uint64_t __sanitizer_unaligned_load64(const void *p); +uint64_t SANITIZER_CDECL __sanitizer_unaligned_load64(const void *p); /// Stores a 16-bit unaligned value. /// /// \param p Pointer to unaligned memory. /// \param x 16-bit value to store. -void __sanitizer_unaligned_store16(void *p, uint16_t x); +void SANITIZER_CDECL __sanitizer_unaligned_store16(void *p, uint16_t x); /// Stores a 32-bit unaligned value. /// /// \param p Pointer to unaligned memory. /// \param x 32-bit value to store. -void __sanitizer_unaligned_store32(void *p, uint32_t x); +void SANITIZER_CDECL __sanitizer_unaligned_store32(void *p, uint32_t x); /// Stores a 64-bit unaligned value. /// /// \param p Pointer to unaligned memory. /// \param x 64-bit value to store. -void __sanitizer_unaligned_store64(void *p, uint64_t x); +void SANITIZER_CDECL __sanitizer_unaligned_store64(void *p, uint64_t x); // Returns 1 on the first call, then returns 0 thereafter. Called by the tool // to ensure only one report is printed when multiple errors occur // simultaneously. -int __sanitizer_acquire_crash_state(); +int SANITIZER_CDECL __sanitizer_acquire_crash_state(); /// Annotates the current state of a contiguous container, such as /// <c>std::vector</c>, <c>std::string</c>, or similar. @@ -129,35 +134,30 @@ int __sanitizer_acquire_crash_state(); /// state <c>mid == end</c>, so that should be the final state when the /// container is destroyed or when the container reallocates the storage. /// -/// For ASan, <c><i>beg</i></c> should be 8-aligned and <c><i>end</i></c> -/// should be either 8-aligned or it should point to the end of a separate -/// heap-, stack-, or global-allocated buffer. So the following example will -/// not work: +/// For ASan, <c><i>beg</i></c> no longer needs to be 8-aligned, +/// first and last granule may be shared with other objects +/// and therefore the function can be used for any allocator. /// -/// \code -/// int64_t x[2]; // 16 bytes, 8-aligned -/// char *beg = (char *)&x[0]; -/// char *end = beg + 12; // Not 8-aligned, not the end of the buffer -/// \endcode +/// The following example shows how to use the function: /// -/// The following, however, will work: /// \code -/// int32_t x[3]; // 12 bytes, but 8-aligned under ASan. +/// int32_t x[3]; // 12 bytes /// char *beg = (char*)&x[0]; -/// char *end = beg + 12; // Not 8-aligned, but is the end of the buffer +/// char *end = beg + 12; +/// __sanitizer_annotate_contiguous_container(beg, end, beg, end); /// \endcode /// /// \note Use this function with caution and do not use for anything other /// than vector-like classes. +/// \note Unaligned <c><i>beg</i></c> or <c><i>end</i></c> may miss bugs in +/// these granules. /// /// \param beg Beginning of memory region. /// \param end End of memory region. /// \param old_mid Old middle of memory region. /// \param new_mid New middle of memory region. -void __sanitizer_annotate_contiguous_container(const void *beg, - const void *end, - const void *old_mid, - const void *new_mid); +void SANITIZER_CDECL __sanitizer_annotate_contiguous_container( + const void *beg, const void *end, const void *old_mid, const void *new_mid); /// Similar to <c>__sanitizer_annotate_contiguous_container</c>. /// @@ -188,7 +188,7 @@ void __sanitizer_annotate_contiguous_container(const void *beg, /// \param old_container_end End of used region. /// \param new_container_beg New beginning of used region. /// \param new_container_end New end of used region. -void __sanitizer_annotate_double_ended_contiguous_container( +void SANITIZER_CDECL __sanitizer_annotate_double_ended_contiguous_container( const void *storage_beg, const void *storage_end, const void *old_container_beg, const void *old_container_end, const void *new_container_beg, const void *new_container_end); @@ -209,8 +209,9 @@ void __sanitizer_annotate_double_ended_contiguous_container( /// /// \returns True if the contiguous container <c>[beg, end)</c> is properly /// poisoned. -int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, - const void *end); +int SANITIZER_CDECL __sanitizer_verify_contiguous_container(const void *beg, + const void *mid, + const void *end); /// Returns true if the double ended contiguous /// container <c>[storage_beg, storage_end)</c> is properly poisoned. @@ -233,7 +234,7 @@ int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, /// \returns True if the double-ended contiguous container <c>[storage_beg, /// container_beg, container_end, end)</c> is properly poisoned - only /// [container_beg; container_end) is addressable. -int __sanitizer_verify_double_ended_contiguous_container( +int SANITIZER_CDECL __sanitizer_verify_double_ended_contiguous_container( const void *storage_beg, const void *container_beg, const void *container_end, const void *storage_end); @@ -247,9 +248,8 @@ int __sanitizer_verify_double_ended_contiguous_container( /// \param end Old end of memory region. /// /// \returns The bad address or NULL. -const void *__sanitizer_contiguous_container_find_bad_address(const void *beg, - const void *mid, - const void *end); +const void *SANITIZER_CDECL __sanitizer_contiguous_container_find_bad_address( + const void *beg, const void *mid, const void *end); /// returns the address of the first improperly poisoned byte. /// @@ -261,13 +261,14 @@ const void *__sanitizer_contiguous_container_find_bad_address(const void *beg, /// \param storage_end End of memory region. /// /// \returns The bad address or NULL. -const void *__sanitizer_double_ended_contiguous_container_find_bad_address( +const void *SANITIZER_CDECL +__sanitizer_double_ended_contiguous_container_find_bad_address( const void *storage_beg, const void *container_beg, const void *container_end, const void *storage_end); /// Prints the stack trace leading to this call (useful for calling from the /// debugger). -void __sanitizer_print_stack_trace(void); +void SANITIZER_CDECL __sanitizer_print_stack_trace(void); // Symbolizes the supplied 'pc' using the format string 'fmt'. // Outputs at most 'out_buf_size' bytes into 'out_buf'. @@ -279,17 +280,20 @@ void __sanitizer_print_stack_trace(void); // Inlined frames can be removed with 'symbolize_inline_frames=0'. // The format syntax is described in // lib/sanitizer_common/sanitizer_stacktrace_printer.h. -void __sanitizer_symbolize_pc(void *pc, const char *fmt, char *out_buf, - size_t out_buf_size); +void SANITIZER_CDECL __sanitizer_symbolize_pc(void *pc, const char *fmt, + char *out_buf, + size_t out_buf_size); // Same as __sanitizer_symbolize_pc, but for data section (i.e. globals). -void __sanitizer_symbolize_global(void *data_ptr, const char *fmt, - char *out_buf, size_t out_buf_size); +void SANITIZER_CDECL __sanitizer_symbolize_global(void *data_ptr, + const char *fmt, + char *out_buf, + size_t out_buf_size); // Determine the return address. #if !defined(_MSC_VER) || defined(__clang__) #define __sanitizer_return_address() \ __builtin_extract_return_addr(__builtin_return_address(0)) #else -extern "C" void *_ReturnAddress(void); +void *SANITIZER_CDECL _ReturnAddress(void); #pragma intrinsic(_ReturnAddress) #define __sanitizer_return_address() _ReturnAddress() #endif @@ -299,8 +303,7 @@ extern "C" void *_ReturnAddress(void); /// Passing 0 will unset the callback. /// /// \param callback User-provided callback. -void __sanitizer_set_death_callback(void (*callback)(void)); - +void SANITIZER_CDECL __sanitizer_set_death_callback(void (*callback)(void)); // Interceptor hooks. // Whenever a libc function interceptor is called, it checks if the @@ -316,8 +319,10 @@ void __sanitizer_set_death_callback(void (*callback)(void)); /// \param s2 Pointer to block of memory. /// \param n Number of bytes to compare. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, - const void *s2, size_t n, int result); +void SANITIZER_CDECL __sanitizer_weak_hook_memcmp(void *called_pc, + const void *s1, + const void *s2, size_t n, + int result); /// Interceptor hook for <c>strncmp()</c>. /// @@ -326,8 +331,10 @@ void __sanitizer_weak_hook_memcmp(void *called_pc, const void *s1, /// \param s2 Pointer to block of memory. /// \param n Number of bytes to compare. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result); +void SANITIZER_CDECL __sanitizer_weak_hook_strncmp(void *called_pc, + const char *s1, + const char *s2, size_t n, + int result); /// Interceptor hook for <c>strncasecmp()</c>. /// @@ -336,8 +343,10 @@ void __sanitizer_weak_hook_strncmp(void *called_pc, const char *s1, /// \param s2 Pointer to block of memory. /// \param n Number of bytes to compare. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, - const char *s2, size_t n, int result); +void SANITIZER_CDECL __sanitizer_weak_hook_strncasecmp(void *called_pc, + const char *s1, + const char *s2, size_t n, + int result); /// Interceptor hook for <c>strcmp()</c>. /// @@ -345,8 +354,9 @@ void __sanitizer_weak_hook_strncasecmp(void *called_pc, const char *s1, /// \param s1 Pointer to block of memory. /// \param s2 Pointer to block of memory. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, - const char *s2, int result); +void SANITIZER_CDECL __sanitizer_weak_hook_strcmp(void *called_pc, + const char *s1, + const char *s2, int result); /// Interceptor hook for <c>strcasecmp()</c>. /// @@ -354,8 +364,10 @@ void __sanitizer_weak_hook_strcmp(void *called_pc, const char *s1, /// \param s1 Pointer to block of memory. /// \param s2 Pointer to block of memory. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, - const char *s2, int result); +void SANITIZER_CDECL __sanitizer_weak_hook_strcasecmp(void *called_pc, + const char *s1, + const char *s2, + int result); /// Interceptor hook for <c>strstr()</c>. /// @@ -363,23 +375,27 @@ void __sanitizer_weak_hook_strcasecmp(void *called_pc, const char *s1, /// \param s1 Pointer to block of memory. /// \param s2 Pointer to block of memory. /// \param result Value returned by the intercepted function. -void __sanitizer_weak_hook_strstr(void *called_pc, const char *s1, - const char *s2, char *result); +void SANITIZER_CDECL __sanitizer_weak_hook_strstr(void *called_pc, + const char *s1, + const char *s2, char *result); -void __sanitizer_weak_hook_strcasestr(void *called_pc, const char *s1, - const char *s2, char *result); +void SANITIZER_CDECL __sanitizer_weak_hook_strcasestr(void *called_pc, + const char *s1, + const char *s2, + char *result); -void __sanitizer_weak_hook_memmem(void *called_pc, - const void *s1, size_t len1, - const void *s2, size_t len2, void *result); +void SANITIZER_CDECL __sanitizer_weak_hook_memmem(void *called_pc, + const void *s1, size_t len1, + const void *s2, size_t len2, + void *result); // Prints stack traces for all live heap allocations ordered by total // allocation size until top_percent of total live heap is shown. top_percent // should be between 1 and 100. At most max_number_of_contexts contexts // (stack traces) are printed. // Experimental feature currently available only with ASan on Linux/x86_64. -void __sanitizer_print_memory_profile(size_t top_percent, - size_t max_number_of_contexts); +void SANITIZER_CDECL __sanitizer_print_memory_profile( + size_t top_percent, size_t max_number_of_contexts); /// Notify ASan that a fiber switch has started (required only if implementing /// your own fiber library). @@ -408,8 +424,9 @@ void __sanitizer_print_memory_profile(size_t top_percent, /// \param[out] fake_stack_save Fake stack save location. /// \param bottom Bottom address of stack. /// \param size Size of stack in bytes. -void __sanitizer_start_switch_fiber(void **fake_stack_save, - const void *bottom, size_t size); +void SANITIZER_CDECL __sanitizer_start_switch_fiber(void **fake_stack_save, + const void *bottom, + size_t size); /// Notify ASan that a fiber switch has completed (required only if /// implementing your own fiber library). @@ -422,18 +439,17 @@ void __sanitizer_start_switch_fiber(void **fake_stack_save, /// \param fake_stack_save Fake stack save location. /// \param[out] bottom_old Bottom address of old stack. /// \param[out] size_old Size of old stack in bytes. -void __sanitizer_finish_switch_fiber(void *fake_stack_save, - const void **bottom_old, - size_t *size_old); +void SANITIZER_CDECL __sanitizer_finish_switch_fiber(void *fake_stack_save, + const void **bottom_old, + size_t *size_old); // Get full module name and calculate pc offset within it. // Returns 1 if pc belongs to some module, 0 if module was not found. -int __sanitizer_get_module_and_offset_for_pc(void *pc, char *module_path, - size_t module_path_len, - void **pc_offset); +int SANITIZER_CDECL __sanitizer_get_module_and_offset_for_pc( + void *pc, char *module_path, size_t module_path_len, void **pc_offset); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_COMMON_INTERFACE_DEFS_H +#endif // SANITIZER_COMMON_INTERFACE_DEFS_H diff --git a/libsanitizer/include/sanitizer/coverage_interface.h b/libsanitizer/include/sanitizer/coverage_interface.h index c063cfe60c5..6235dfc2d4b 100644 --- a/libsanitizer/include/sanitizer/coverage_interface.h +++ b/libsanitizer/include/sanitizer/coverage_interface.h @@ -18,18 +18,19 @@ extern "C" { #endif - // Record and dump coverage info. - void __sanitizer_cov_dump(void); +// Record and dump coverage info. +void SANITIZER_CDECL __sanitizer_cov_dump(void); - // Clear collected coverage info. - void __sanitizer_cov_reset(void); +// Clear collected coverage info. +void SANITIZER_CDECL __sanitizer_cov_reset(void); - // Dump collected coverage info. Sorts pcs by module into individual .sancov - // files. - void __sanitizer_dump_coverage(const uintptr_t *pcs, uintptr_t len); +// Dump collected coverage info. Sorts pcs by module into individual .sancov +// files. +void SANITIZER_CDECL __sanitizer_dump_coverage(const uintptr_t *pcs, + uintptr_t len); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_COVERAG_INTERFACE_H +#endif // SANITIZER_COVERAG_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/dfsan_interface.h b/libsanitizer/include/sanitizer/dfsan_interface.h index 519bfffa9a2..4e52e1b54cd 100644 --- a/libsanitizer/include/sanitizer/dfsan_interface.h +++ b/libsanitizer/include/sanitizer/dfsan_interface.h @@ -13,9 +13,9 @@ #ifndef DFSAN_INTERFACE_H #define DFSAN_INTERFACE_H +#include <sanitizer/common_interface_defs.h> #include <stddef.h> #include <stdint.h> -#include <sanitizer/common_interface_defs.h> #ifdef __cplusplus extern "C" { @@ -25,29 +25,30 @@ typedef uint8_t dfsan_label; typedef uint32_t dfsan_origin; /// Signature of the callback argument to dfsan_set_write_callback(). -typedef void (*dfsan_write_callback_t)(int fd, const void *buf, size_t count); +typedef void(SANITIZER_CDECL *dfsan_write_callback_t)(int fd, const void *buf, + size_t count); /// Signature of the callback argument to dfsan_set_conditional_callback(). -typedef void (*dfsan_conditional_callback_t)(dfsan_label label, - dfsan_origin origin); +typedef void(SANITIZER_CDECL *dfsan_conditional_callback_t)( + dfsan_label label, dfsan_origin origin); /// Signature of the callback argument to dfsan_set_reaches_function_callback(). /// The description is intended to hold the name of the variable. -typedef void (*dfsan_reaches_function_callback_t)(dfsan_label label, - dfsan_origin origin, - const char *file, - unsigned int line, - const char *function); +typedef void(SANITIZER_CDECL *dfsan_reaches_function_callback_t)( + dfsan_label label, dfsan_origin origin, const char *file, unsigned int line, + const char *function); /// Computes the union of \c l1 and \c l2, resulting in a union label. -dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2); +dfsan_label SANITIZER_CDECL dfsan_union(dfsan_label l1, dfsan_label l2); /// Sets the label for each address in [addr,addr+size) to \c label. -void dfsan_set_label(dfsan_label label, void *addr, size_t size); +void SANITIZER_CDECL dfsan_set_label(dfsan_label label, void *addr, + size_t size); /// Sets the label for each address in [addr,addr+size) to the union of the /// current label for that address and \c label. -void dfsan_add_label(dfsan_label label, void *addr, size_t size); +void SANITIZER_CDECL dfsan_add_label(dfsan_label label, void *addr, + size_t size); /// Retrieves the label associated with the given data. /// @@ -55,23 +56,24 @@ void dfsan_add_label(dfsan_label label, void *addr, size_t size); /// which can be truncated or extended (implicitly or explicitly) as necessary. /// The truncation/extension operations will preserve the label of the original /// value. -dfsan_label dfsan_get_label(long data); +dfsan_label SANITIZER_CDECL dfsan_get_label(long data); /// Retrieves the immediate origin associated with the given data. The returned /// origin may point to another origin. /// /// The type of 'data' is arbitrary. -dfsan_origin dfsan_get_origin(long data); +dfsan_origin SANITIZER_CDECL dfsan_get_origin(long data); /// Retrieves the label associated with the data at the given address. -dfsan_label dfsan_read_label(const void *addr, size_t size); +dfsan_label SANITIZER_CDECL dfsan_read_label(const void *addr, size_t size); /// Return the origin associated with the first taint byte in the size bytes /// from the address addr. -dfsan_origin dfsan_read_origin_of_first_taint(const void *addr, size_t size); +dfsan_origin SANITIZER_CDECL dfsan_read_origin_of_first_taint(const void *addr, + size_t size); -/// Returns whether the given label label contains the label elem. -int dfsan_has_label(dfsan_label label, dfsan_label elem); +/// Returns whether the given label contains the label elem. +int SANITIZER_CDECL dfsan_has_label(dfsan_label label, dfsan_label elem); /// Flushes the DFSan shadow, i.e. forgets about all labels currently associated /// with the application memory. Use this call to start over the taint tracking @@ -79,37 +81,39 @@ int dfsan_has_label(dfsan_label label, dfsan_label elem); /// /// Note: If another thread is working with tainted data during the flush, that /// taint could still be written to shadow after the flush. -void dfsan_flush(void); +void SANITIZER_CDECL dfsan_flush(void); /// Sets a callback to be invoked on calls to write(). The callback is invoked /// before the write is done. The write is not guaranteed to succeed when the /// callback executes. Pass in NULL to remove any callback. -void dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback); +void SANITIZER_CDECL +dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback); /// Sets a callback to be invoked on any conditional expressions which have a /// taint label set. This can be used to find where tainted data influences /// the behavior of the program. /// These callbacks will only be added when -dfsan-conditional-callbacks=true. -void dfsan_set_conditional_callback(dfsan_conditional_callback_t callback); +void SANITIZER_CDECL +dfsan_set_conditional_callback(dfsan_conditional_callback_t callback); /// Conditional expressions occur during signal handlers. /// Making callbacks that handle signals well is tricky, so when /// -dfsan-conditional-callbacks=true, conditional expressions used in signal /// handlers will add the labels they see into a global (bitwise-or together). /// This function returns all label bits seen in signal handler conditions. -dfsan_label dfsan_get_labels_in_signal_conditional(); +dfsan_label SANITIZER_CDECL dfsan_get_labels_in_signal_conditional(); /// Sets a callback to be invoked when tainted data reaches a function. /// This could occur at function entry, or at a load instruction. /// These callbacks will only be added if -dfsan-reaches-function-callbacks=1. -void dfsan_set_reaches_function_callback( - dfsan_reaches_function_callback_t callback); +void SANITIZER_CDECL +dfsan_set_reaches_function_callback(dfsan_reaches_function_callback_t callback); /// Making callbacks that handle signals well is tricky, so when /// -dfsan-reaches-function-callbacks=true, functions reached in signal /// handlers will add the labels they see into a global (bitwise-or together). /// This function returns all label bits seen during signal handlers. -dfsan_label dfsan_get_labels_in_signal_reaches_function(); +dfsan_label SANITIZER_CDECL dfsan_get_labels_in_signal_reaches_function(); /// Interceptor hooks. /// Whenever a dfsan's custom function is called the corresponding @@ -117,20 +121,25 @@ dfsan_label dfsan_get_labels_in_signal_reaches_function(); /// The primary use case is taint-guided fuzzing, where the fuzzer /// needs to see the parameters of the function and the labels. /// FIXME: implement more hooks. -void dfsan_weak_hook_memcmp(void *caller_pc, const void *s1, const void *s2, - size_t n, dfsan_label s1_label, - dfsan_label s2_label, dfsan_label n_label); -void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2, - size_t n, dfsan_label s1_label, - dfsan_label s2_label, dfsan_label n_label); +void SANITIZER_CDECL dfsan_weak_hook_memcmp(void *caller_pc, const void *s1, + const void *s2, size_t n, + dfsan_label s1_label, + dfsan_label s2_label, + dfsan_label n_label); +void SANITIZER_CDECL dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, + const char *s2, size_t n, + dfsan_label s1_label, + dfsan_label s2_label, + dfsan_label n_label); /// Prints the origin trace of the label at the address addr to stderr. It also /// prints description at the beginning of the trace. If origin tracking is not /// on, or the address is not labeled, it prints nothing. -void dfsan_print_origin_trace(const void *addr, const char *description); +void SANITIZER_CDECL dfsan_print_origin_trace(const void *addr, + const char *description); /// As above, but use an origin id from dfsan_get_origin() instead of address. /// Does not include header line with taint label and address information. -void dfsan_print_origin_id_trace(dfsan_origin origin); +void SANITIZER_CDECL dfsan_print_origin_id_trace(dfsan_origin origin); /// Prints the origin trace of the label at the address \p addr to a /// pre-allocated output buffer. If origin tracking is not on, or the address is @@ -166,12 +175,15 @@ void dfsan_print_origin_id_trace(dfsan_origin origin); /// \returns The number of symbols that should have been written to \p out_buf /// (not including trailing null byte '\0'). Thus, the string is truncated iff /// return value is not less than \p out_buf_size. -size_t dfsan_sprint_origin_trace(const void *addr, const char *description, - char *out_buf, size_t out_buf_size); +size_t SANITIZER_CDECL dfsan_sprint_origin_trace(const void *addr, + const char *description, + char *out_buf, + size_t out_buf_size); /// As above, but use an origin id from dfsan_get_origin() instead of address. /// Does not include header line with taint label and address information. -size_t dfsan_sprint_origin_id_trace(dfsan_origin origin, char *out_buf, - size_t out_buf_size); +size_t SANITIZER_CDECL dfsan_sprint_origin_id_trace(dfsan_origin origin, + char *out_buf, + size_t out_buf_size); /// Prints the stack trace leading to this call to a pre-allocated output /// buffer. @@ -184,19 +196,20 @@ size_t dfsan_sprint_origin_id_trace(dfsan_origin origin, char *out_buf, /// \returns The number of symbols that should have been written to \p out_buf /// (not including trailing null byte '\0'). Thus, the string is truncated iff /// return value is not less than \p out_buf_size. -size_t dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size); +size_t SANITIZER_CDECL dfsan_sprint_stack_trace(char *out_buf, + size_t out_buf_size); /// Retrieves the very first origin associated with the data at the given /// address. -dfsan_origin dfsan_get_init_origin(const void *addr); +dfsan_origin SANITIZER_CDECL dfsan_get_init_origin(const void *addr); /// Returns the value of -dfsan-track-origins. /// * 0: do not track origins. /// * 1: track origins at memory store operations. /// * 2: track origins at memory load and store operations. -int dfsan_get_track_origins(void); +int SANITIZER_CDECL dfsan_get_track_origins(void); #ifdef __cplusplus -} // extern "C" +} // extern "C" template <typename T> void dfsan_set_label(dfsan_label label, T &data) { dfsan_set_label(label, (void *)&data, sizeof(T)); @@ -204,4 +217,4 @@ template <typename T> void dfsan_set_label(dfsan_label label, T &data) { #endif -#endif // DFSAN_INTERFACE_H +#endif // DFSAN_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/hwasan_interface.h b/libsanitizer/include/sanitizer/hwasan_interface.h index ee742c7f303..abe310c0666 100644 --- a/libsanitizer/include/sanitizer/hwasan_interface.h +++ b/libsanitizer/include/sanitizer/hwasan_interface.h @@ -18,82 +18,88 @@ #ifdef __cplusplus extern "C" { #endif - // Libc hook for program startup in statically linked executables. - // Initializes enough of the runtime to run instrumented code. This function - // should only be called in statically linked executables because it modifies - // the GOT, which won't work in regular binaries because RELRO will already - // have been applied by the time the function is called. This also means that - // the function should be called before libc applies RELRO. - // Does not call libc unless there is an error. - // Can be called multiple times. - void __hwasan_init_static(void); - - // This function may be optionally provided by user and should return - // a string containing HWASan runtime options. See asan_flags.h for details. - const char* __hwasan_default_options(void); - - void __hwasan_enable_allocator_tagging(void); - void __hwasan_disable_allocator_tagging(void); - - // Mark region of memory with the given tag. Both address and size need to be - // 16-byte aligned. - void __hwasan_tag_memory(const volatile void *p, unsigned char tag, - size_t size); - - /// Set pointer tag. Previous tag is lost. - void *__hwasan_tag_pointer(const volatile void *p, unsigned char tag); - - // Set memory tag from the current SP address to the given address to zero. - // This is meant to annotate longjmp and other non-local jumps. - // This function needs to know the (almost) exact destination frame address; - // clearing shadow for the entire thread stack like __asan_handle_no_return - // does would cause false reports. - void __hwasan_handle_longjmp(const void *sp_dst); - - // Set memory tag for the part of the current thread stack below sp_dst to - // zero. Call this in vfork() before returning in the parent process. - void __hwasan_handle_vfork(const void *sp_dst); - - // Libc hook for thread creation. Should be called in the child thread before - // any instrumented code. - void __hwasan_thread_enter(); - - // Libc hook for thread destruction. No instrumented code should run after - // this call. - void __hwasan_thread_exit(); - - // Print shadow and origin for the memory range to stderr in a human-readable - // format. - void __hwasan_print_shadow(const volatile void *x, size_t size); - - // Print one-line report about the memory usage of the current process. - void __hwasan_print_memory_usage(); - - /* Returns the offset of the first byte in the memory range that can not be - * accessed through the pointer in x, or -1 if the whole range is good. */ - intptr_t __hwasan_test_shadow(const volatile void *x, size_t size); - - /* Sets the callback function to be called during HWASan error reporting. */ - void __hwasan_set_error_report_callback(void (*callback)(const char *)); - - int __sanitizer_posix_memalign(void **memptr, size_t alignment, size_t size); - void * __sanitizer_memalign(size_t alignment, size_t size); - void * __sanitizer_aligned_alloc(size_t alignment, size_t size); - void * __sanitizer___libc_memalign(size_t alignment, size_t size); - void * __sanitizer_valloc(size_t size); - void * __sanitizer_pvalloc(size_t size); - void __sanitizer_free(void *ptr); - void __sanitizer_cfree(void *ptr); - size_t __sanitizer_malloc_usable_size(const void *ptr); - struct mallinfo __sanitizer_mallinfo(); - int __sanitizer_mallopt(int cmd, int value); - void __sanitizer_malloc_stats(void); - void * __sanitizer_calloc(size_t nmemb, size_t size); - void * __sanitizer_realloc(void *ptr, size_t size); - void * __sanitizer_reallocarray(void *ptr, size_t nmemb, size_t size); - void * __sanitizer_malloc(size_t size); +// Libc hook for program startup in statically linked executables. +// Initializes enough of the runtime to run instrumented code. This function +// should only be called in statically linked executables because it modifies +// the GOT, which won't work in regular binaries because RELRO will already +// have been applied by the time the function is called. This also means that +// the function should be called before libc applies RELRO. +// Does not call libc unless there is an error. +// Can be called multiple times. +void SANITIZER_CDECL __hwasan_init_static(void); + +// This function may be optionally provided by user and should return +// a string containing HWASan runtime options. See asan_flags.h for details. +const char *SANITIZER_CDECL __hwasan_default_options(void); + +void SANITIZER_CDECL __hwasan_enable_allocator_tagging(void); +void SANITIZER_CDECL __hwasan_disable_allocator_tagging(void); + +// Mark region of memory with the given tag. Both address and size need to be +// 16-byte aligned. +void SANITIZER_CDECL __hwasan_tag_memory(const volatile void *p, + unsigned char tag, size_t size); + +/// Set pointer tag. Previous tag is lost. +void *SANITIZER_CDECL __hwasan_tag_pointer(const volatile void *p, + unsigned char tag); + +// Set memory tag from the current SP address to the given address to zero. +// This is meant to annotate longjmp and other non-local jumps. +// This function needs to know the (almost) exact destination frame address; +// clearing shadow for the entire thread stack like __asan_handle_no_return +// does would cause false reports. +void SANITIZER_CDECL __hwasan_handle_longjmp(const void *sp_dst); + +// Set memory tag for the part of the current thread stack below sp_dst to +// zero. Call this in vfork() before returning in the parent process. +void SANITIZER_CDECL __hwasan_handle_vfork(const void *sp_dst); + +// Libc hook for thread creation. Should be called in the child thread before +// any instrumented code. +void SANITIZER_CDECL __hwasan_thread_enter(); + +// Libc hook for thread destruction. No instrumented code should run after +// this call. +void SANITIZER_CDECL __hwasan_thread_exit(); + +// Print shadow and origin for the memory range to stderr in a human-readable +// format. +void SANITIZER_CDECL __hwasan_print_shadow(const volatile void *x, size_t size); + +// Print one-line report about the memory usage of the current process. +void SANITIZER_CDECL __hwasan_print_memory_usage(); + +/* Returns the offset of the first byte in the memory range that can not be + * accessed through the pointer in x, or -1 if the whole range is good. */ +intptr_t SANITIZER_CDECL __hwasan_test_shadow(const volatile void *x, + size_t size); + +/* Sets the callback function to be called during HWASan error reporting. */ +void SANITIZER_CDECL +__hwasan_set_error_report_callback(void (*callback)(const char *)); + +int SANITIZER_CDECL __sanitizer_posix_memalign(void **memptr, size_t alignment, + size_t size); +void *SANITIZER_CDECL __sanitizer_memalign(size_t alignment, size_t size); +void *SANITIZER_CDECL __sanitizer_aligned_alloc(size_t alignment, size_t size); +void *SANITIZER_CDECL __sanitizer___libc_memalign(size_t alignment, + size_t size); +void *SANITIZER_CDECL __sanitizer_valloc(size_t size); +void *SANITIZER_CDECL __sanitizer_pvalloc(size_t size); +void SANITIZER_CDECL __sanitizer_free(void *ptr); +void SANITIZER_CDECL __sanitizer_cfree(void *ptr); +size_t SANITIZER_CDECL __sanitizer_malloc_usable_size(const void *ptr); +struct mallinfo SANITIZER_CDECL __sanitizer_mallinfo(); +int SANITIZER_CDECL __sanitizer_mallopt(int cmd, int value); +void SANITIZER_CDECL __sanitizer_malloc_stats(void); +void *SANITIZER_CDECL __sanitizer_calloc(size_t nmemb, size_t size); +void *SANITIZER_CDECL __sanitizer_realloc(void *ptr, size_t size); +void *SANITIZER_CDECL __sanitizer_reallocarray(void *ptr, size_t nmemb, + size_t size); +void *SANITIZER_CDECL __sanitizer_malloc(size_t size); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_HWASAN_INTERFACE_H +#endif // SANITIZER_HWASAN_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/lsan_interface.h b/libsanitizer/include/sanitizer/lsan_interface.h index 2bb992672f2..18f3456a126 100644 --- a/libsanitizer/include/sanitizer/lsan_interface.h +++ b/libsanitizer/include/sanitizer/lsan_interface.h @@ -18,72 +18,72 @@ #ifdef __cplusplus extern "C" { #endif - // Allocations made between calls to __lsan_disable() and __lsan_enable() will - // be treated as non-leaks. Disable/enable pairs may be nested. - void __lsan_disable(void); - void __lsan_enable(void); +// Allocations made between calls to __lsan_disable() and __lsan_enable() will +// be treated as non-leaks. Disable/enable pairs may be nested. +void SANITIZER_CDECL __lsan_disable(void); +void SANITIZER_CDECL __lsan_enable(void); - // The heap object into which p points will be treated as a non-leak. - void __lsan_ignore_object(const void *p); +// The heap object into which p points will be treated as a non-leak. +void SANITIZER_CDECL __lsan_ignore_object(const void *p); - // Memory regions registered through this interface will be treated as sources - // of live pointers during leak checking. Useful if you store pointers in - // mapped memory. - // Points of note: - // - __lsan_unregister_root_region() must be called with the same pointer and - // size that have earlier been passed to __lsan_register_root_region() - // - LSan will skip any inaccessible memory when scanning a root region. E.g., - // if you map memory within a larger region that you have mprotect'ed, you can - // register the entire large region. - // - the implementation is not optimized for performance. This interface is - // intended to be used for a small number of relatively static regions. - void __lsan_register_root_region(const void *p, size_t size); - void __lsan_unregister_root_region(const void *p, size_t size); +// Memory regions registered through this interface will be treated as sources +// of live pointers during leak checking. Useful if you store pointers in +// mapped memory. +// Points of note: +// - __lsan_unregister_root_region() must be called with the same pointer and +// size that have earlier been passed to __lsan_register_root_region() +// - LSan will skip any inaccessible memory when scanning a root region. E.g., +// if you map memory within a larger region that you have mprotect'ed, you can +// register the entire large region. +// - the implementation is not optimized for performance. This interface is +// intended to be used for a small number of relatively static regions. +void SANITIZER_CDECL __lsan_register_root_region(const void *p, size_t size); +void SANITIZER_CDECL __lsan_unregister_root_region(const void *p, size_t size); - // Check for leaks now. This function behaves identically to the default - // end-of-process leak check. In particular, it will terminate the process if - // leaks are found and the exitcode runtime flag is non-zero. - // Subsequent calls to this function will have no effect and end-of-process - // leak check will not run. Effectively, end-of-process leak check is moved to - // the time of first invocation of this function. - // By calling this function early during process shutdown, you can instruct - // LSan to ignore shutdown-only leaks which happen later on. - void __lsan_do_leak_check(void); +// Check for leaks now. This function behaves identically to the default +// end-of-process leak check. In particular, it will terminate the process if +// leaks are found and the exitcode runtime flag is non-zero. +// Subsequent calls to this function will have no effect and end-of-process +// leak check will not run. Effectively, end-of-process leak check is moved to +// the time of first invocation of this function. +// By calling this function early during process shutdown, you can instruct +// LSan to ignore shutdown-only leaks which happen later on. +void SANITIZER_CDECL __lsan_do_leak_check(void); - // Check for leaks now. Returns zero if no leaks have been found or if leak - // detection is disabled, non-zero otherwise. - // This function may be called repeatedly, e.g. to periodically check a - // long-running process. It prints a leak report if appropriate, but does not - // terminate the process. It does not affect the behavior of - // __lsan_do_leak_check() or the end-of-process leak check, and is not - // affected by them. - int __lsan_do_recoverable_leak_check(void); +// Check for leaks now. Returns zero if no leaks have been found or if leak +// detection is disabled, non-zero otherwise. +// This function may be called repeatedly, e.g. to periodically check a +// long-running process. It prints a leak report if appropriate, but does not +// terminate the process. It does not affect the behavior of +// __lsan_do_leak_check() or the end-of-process leak check, and is not +// affected by them. +int SANITIZER_CDECL __lsan_do_recoverable_leak_check(void); - // The user may optionally provide this function to disallow leak checking - // for the program it is linked into (if the return value is non-zero). This - // function must be defined as returning a constant value; any behavior beyond - // that is unsupported. - // To avoid dead stripping, you may need to define this function with - // __attribute__((used)) - int __lsan_is_turned_off(void); +// The user may optionally provide this function to disallow leak checking +// for the program it is linked into (if the return value is non-zero). This +// function must be defined as returning a constant value; any behavior beyond +// that is unsupported. +// To avoid dead stripping, you may need to define this function with +// __attribute__((used)) +int SANITIZER_CDECL __lsan_is_turned_off(void); - // This function may be optionally provided by user and should return - // a string containing LSan runtime options. See lsan_flags.inc for details. - const char *__lsan_default_options(void); +// This function may be optionally provided by user and should return +// a string containing LSan runtime options. See lsan_flags.inc for details. +const char *SANITIZER_CDECL __lsan_default_options(void); - // This function may be optionally provided by the user and should return - // a string containing LSan suppressions. - const char *__lsan_default_suppressions(void); +// This function may be optionally provided by the user and should return +// a string containing LSan suppressions. +const char *SANITIZER_CDECL __lsan_default_suppressions(void); #ifdef __cplusplus -} // extern "C" +} // extern "C" namespace __lsan { class ScopedDisabler { - public: +public: ScopedDisabler() { __lsan_disable(); } ~ScopedDisabler() { __lsan_enable(); } }; -} // namespace __lsan +} // namespace __lsan #endif -#endif // SANITIZER_LSAN_INTERFACE_H +#endif // SANITIZER_LSAN_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/memprof_interface.h b/libsanitizer/include/sanitizer/memprof_interface.h index 76031de4014..fe0a2fc5ef0 100644 --- a/libsanitizer/include/sanitizer/memprof_interface.h +++ b/libsanitizer/include/sanitizer/memprof_interface.h @@ -24,25 +24,26 @@ extern "C" { /// /// \param addr Start of memory region. /// \param size Size of memory region. -void __memprof_record_access_range(void const volatile *addr, size_t size); +void SANITIZER_CDECL __memprof_record_access_range(void const volatile *addr, + size_t size); /// Records access to a memory address <c><i>addr</i></c>. /// /// This memory must be previously allocated by your program. /// /// \param addr Accessed memory address -void __memprof_record_access(void const volatile *addr); +void SANITIZER_CDECL __memprof_record_access(void const volatile *addr); /// User-provided callback on MemProf errors. /// /// You can provide a function that would be called immediately when MemProf /// detects an error. This is useful in cases when MemProf detects an error but /// your program crashes before the MemProf report is printed. -void __memprof_on_error(void); +void SANITIZER_CDECL __memprof_on_error(void); /// Prints accumulated statistics to <c>stderr</c> (useful for calling from the /// debugger). -void __memprof_print_accumulated_stats(void); +void SANITIZER_CDECL __memprof_print_accumulated_stats(void); /// User-provided default option settings. /// @@ -51,12 +52,12 @@ void __memprof_print_accumulated_stats(void); /// <c>verbosity=1:print_stats=1</c>). /// /// \returns Default options string. -const char *__memprof_default_options(void); +const char *SANITIZER_CDECL __memprof_default_options(void); /// Prints the memory profile to the current profile file. /// /// \returns 0 on success. -int __memprof_profile_dump(void); +int SANITIZER_CDECL __memprof_profile_dump(void); #ifdef __cplusplus } // extern "C" diff --git a/libsanitizer/include/sanitizer/msan_interface.h b/libsanitizer/include/sanitizer/msan_interface.h index 854b12cda36..6fedc031254 100644 --- a/libsanitizer/include/sanitizer/msan_interface.h +++ b/libsanitizer/include/sanitizer/msan_interface.h @@ -18,109 +18,118 @@ #ifdef __cplusplus extern "C" { #endif - /* Set raw origin for the memory range. */ - void __msan_set_origin(const volatile void *a, size_t size, uint32_t origin); - - /* Get raw origin for an address. */ - uint32_t __msan_get_origin(const volatile void *a); - - /* Test that this_id is a descendant of prev_id (or they are simply equal). - * "descendant" here means they are part of the same chain, created with - * __msan_chain_origin. */ - int __msan_origin_is_descendant_or_same(uint32_t this_id, uint32_t prev_id); - - /* Returns non-zero if tracking origins. */ - int __msan_get_track_origins(void); - - /* Returns the origin id of the latest UMR in the calling thread. */ - uint32_t __msan_get_umr_origin(void); - - /* Make memory region fully initialized (without changing its contents). */ - void __msan_unpoison(const volatile void *a, size_t size); - - /* Make a null-terminated string fully initialized (without changing its - contents). */ - void __msan_unpoison_string(const volatile char *a); - - /* Make first n parameters of the next function call fully initialized. */ - void __msan_unpoison_param(size_t n); - - /* Make memory region fully uninitialized (without changing its contents). - This is a legacy interface that does not update origin information. Use - __msan_allocated_memory() instead. */ - void __msan_poison(const volatile void *a, size_t size); - - /* Make memory region partially uninitialized (without changing its contents). - */ - void __msan_partial_poison(const volatile void *data, void *shadow, - size_t size); - - /* Returns the offset of the first (at least partially) poisoned byte in the - memory range, or -1 if the whole range is good. */ - intptr_t __msan_test_shadow(const volatile void *x, size_t size); - - /* Checks that memory range is fully initialized, and reports an error if it - * is not. */ - void __msan_check_mem_is_initialized(const volatile void *x, size_t size); - - /* For testing: - __msan_set_expect_umr(1); - ... some buggy code ... - __msan_set_expect_umr(0); - The last line will verify that a UMR happened. */ - void __msan_set_expect_umr(int expect_umr); - - /* Change the value of keep_going flag. Non-zero value means don't terminate - program execution when an error is detected. This will not affect error in - modules that were compiled without the corresponding compiler flag. */ - void __msan_set_keep_going(int keep_going); - - /* Print shadow and origin for the memory range to stderr in a human-readable - format. */ - void __msan_print_shadow(const volatile void *x, size_t size); - - /* Print shadow for the memory range to stderr in a minimalistic - human-readable format. */ - void __msan_dump_shadow(const volatile void *x, size_t size); - - /* Returns true if running under a dynamic tool (DynamoRio-based). */ - int __msan_has_dynamic_component(void); - - /* Tell MSan about newly allocated memory (ex.: custom allocator). - Memory will be marked uninitialized, with origin at the call site. */ - void __msan_allocated_memory(const volatile void* data, size_t size); - - /* Tell MSan about newly destroyed memory. Mark memory as uninitialized. */ - void __sanitizer_dtor_callback(const volatile void* data, size_t size); - void __sanitizer_dtor_callback_fields(const volatile void *data, size_t size); - void __sanitizer_dtor_callback_vptr(const volatile void *data); - - /* This function may be optionally provided by user and should return - a string containing Msan runtime options. See msan_flags.h for details. */ - const char* __msan_default_options(void); - - /* Deprecated. Call __sanitizer_set_death_callback instead. */ - void __msan_set_death_callback(void (*callback)(void)); - - /* Update shadow for the application copy of size bytes from src to dst. - Src and dst are application addresses. This function does not copy the - actual application memory, it only updates shadow and origin for such - copy. Source and destination regions can overlap. */ - void __msan_copy_shadow(const volatile void *dst, const volatile void *src, - size_t size); - - /* Disables uninitialized memory checks in interceptors. */ - void __msan_scoped_disable_interceptor_checks(void); - - /* Re-enables uninitialized memory checks in interceptors after a previous - call to __msan_scoped_disable_interceptor_checks. */ - void __msan_scoped_enable_interceptor_checks(void); - - void __msan_start_switch_fiber(const void *bottom, size_t size); - void __msan_finish_switch_fiber(const void **bottom_old, size_t *size_old); +/* Set raw origin for the memory range. */ +void SANITIZER_CDECL __msan_set_origin(const volatile void *a, size_t size, + uint32_t origin); + +/* Get raw origin for an address. */ +uint32_t SANITIZER_CDECL __msan_get_origin(const volatile void *a); + +/* Test that this_id is a descendant of prev_id (or they are simply equal). + * "descendant" here means they are part of the same chain, created with + * __msan_chain_origin. */ +int SANITIZER_CDECL __msan_origin_is_descendant_or_same(uint32_t this_id, + uint32_t prev_id); + +/* Returns non-zero if tracking origins. */ +int SANITIZER_CDECL __msan_get_track_origins(void); + +/* Returns the origin id of the latest UMR in the calling thread. */ +uint32_t SANITIZER_CDECL __msan_get_umr_origin(void); + +/* Make memory region fully initialized (without changing its contents). */ +void SANITIZER_CDECL __msan_unpoison(const volatile void *a, size_t size); + +/* Make a null-terminated string fully initialized (without changing its + contents). */ +void SANITIZER_CDECL __msan_unpoison_string(const volatile char *a); + +/* Make first n parameters of the next function call fully initialized. */ +void SANITIZER_CDECL __msan_unpoison_param(size_t n); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void SANITIZER_CDECL __msan_poison(const volatile void *a, size_t size); + +/* Make memory region partially uninitialized (without changing its contents). + */ +void SANITIZER_CDECL __msan_partial_poison(const volatile void *data, + void *shadow, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t SANITIZER_CDECL __msan_test_shadow(const volatile void *x, + size_t size); + +/* Checks that memory range is fully initialized, and reports an error if it + * is not. */ +void SANITIZER_CDECL __msan_check_mem_is_initialized(const volatile void *x, + size_t size); + +/* For testing: + __msan_set_expect_umr(1); + ... some buggy code ... + __msan_set_expect_umr(0); + The last line will verify that a UMR happened. */ +void SANITIZER_CDECL __msan_set_expect_umr(int expect_umr); + +/* Change the value of keep_going flag. Non-zero value means don't terminate + program execution when an error is detected. This will not affect error in + modules that were compiled without the corresponding compiler flag. */ +void SANITIZER_CDECL __msan_set_keep_going(int keep_going); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void SANITIZER_CDECL __msan_print_shadow(const volatile void *x, size_t size); + +/* Print shadow for the memory range to stderr in a minimalistic + human-readable format. */ +void SANITIZER_CDECL __msan_dump_shadow(const volatile void *x, size_t size); + +/* Returns true if running under a dynamic tool (DynamoRio-based). */ +int SANITIZER_CDECL __msan_has_dynamic_component(void); + +/* Tell MSan about newly allocated memory (ex.: custom allocator). + Memory will be marked uninitialized, with origin at the call site. */ +void SANITIZER_CDECL __msan_allocated_memory(const volatile void *data, + size_t size); + +/* Tell MSan about newly destroyed memory. Mark memory as uninitialized. */ +void SANITIZER_CDECL __sanitizer_dtor_callback(const volatile void *data, + size_t size); +void SANITIZER_CDECL __sanitizer_dtor_callback_fields(const volatile void *data, + size_t size); +void SANITIZER_CDECL __sanitizer_dtor_callback_vptr(const volatile void *data); + +/* This function may be optionally provided by user and should return + a string containing Msan runtime options. See msan_flags.h for details. */ +const char *SANITIZER_CDECL __msan_default_options(void); + +/* Deprecated. Call __sanitizer_set_death_callback instead. */ +void SANITIZER_CDECL +__msan_set_death_callback(void(SANITIZER_CDECL *callback)(void)); + +/* Update shadow for the application copy of size bytes from src to dst. + Src and dst are application addresses. This function does not copy the + actual application memory, it only updates shadow and origin for such + copy. Source and destination regions can overlap. */ +void SANITIZER_CDECL __msan_copy_shadow(const volatile void *dst, + const volatile void *src, size_t size); + +/* Disables uninitialized memory checks in interceptors. */ +void SANITIZER_CDECL __msan_scoped_disable_interceptor_checks(void); + +/* Re-enables uninitialized memory checks in interceptors after a previous + call to __msan_scoped_disable_interceptor_checks. */ +void SANITIZER_CDECL __msan_scoped_enable_interceptor_checks(void); + +void SANITIZER_CDECL __msan_start_switch_fiber(const void *bottom, size_t size); +void SANITIZER_CDECL __msan_finish_switch_fiber(const void **bottom_old, + size_t *size_old); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif #endif diff --git a/libsanitizer/include/sanitizer/scudo_interface.h b/libsanitizer/include/sanitizer/scudo_interface.h index dd522c1efc2..37fd7bfeccf 100644 --- a/libsanitizer/include/sanitizer/scudo_interface.h +++ b/libsanitizer/include/sanitizer/scudo_interface.h @@ -17,22 +17,22 @@ #ifdef __cplusplus extern "C" { #endif - // This function may be optionally provided by a user and should return - // a string containing Scudo runtime options. See scudo_flags.h for details. - const char* __scudo_default_options(void); +// This function may be optionally provided by a user and should return +// a string containing Scudo runtime options. See scudo_flags.h for details. +const char *SANITIZER_CDECL __scudo_default_options(void); - // This function allows to set the RSS limit at runtime. This can be either - // the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit - // can be removed by setting LimitMb to 0. This function's parameters should - // be fully trusted to avoid security mishaps. - void __scudo_set_rss_limit(size_t LimitMb, int HardLimit); +// This function allows to set the RSS limit at runtime. This can be either +// the hard limit (HardLimit=1) or the soft limit (HardLimit=0). The limit +// can be removed by setting LimitMb to 0. This function's parameters should +// be fully trusted to avoid security mishaps. +void SANITIZER_CDECL __scudo_set_rss_limit(size_t LimitMb, int HardLimit); - // This function outputs various allocator statistics for both the Primary - // and Secondary allocators, including memory usage, number of allocations - // and deallocations. - void __scudo_print_stats(void); +// This function outputs various allocator statistics for both the Primary +// and Secondary allocators, including memory usage, number of allocations +// and deallocations. +void SANITIZER_CDECL __scudo_print_stats(void); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_SCUDO_INTERFACE_H_ +#endif // SANITIZER_SCUDO_INTERFACE_H_ diff --git a/libsanitizer/include/sanitizer/tsan_interface.h b/libsanitizer/include/sanitizer/tsan_interface.h index 58f2513734e..e11a4175cd8 100644 --- a/libsanitizer/include/sanitizer/tsan_interface.h +++ b/libsanitizer/include/sanitizer/tsan_interface.h @@ -21,8 +21,8 @@ extern "C" { // __tsan_release establishes a happens-before relation with a preceding // __tsan_acquire on the same address. -void __tsan_acquire(void *addr); -void __tsan_release(void *addr); +void SANITIZER_CDECL __tsan_acquire(void *addr); +void SANITIZER_CDECL __tsan_release(void *addr); // Annotations for custom mutexes. // The annotations allow to get better reports (with sets of locked mutexes), @@ -52,16 +52,16 @@ static const unsigned __tsan_mutex_not_static = 1 << 8; // Mutex operation flags: // Denotes read lock operation. -static const unsigned __tsan_mutex_read_lock = 1 << 3; +static const unsigned __tsan_mutex_read_lock = 1 << 3; // Denotes try lock operation. -static const unsigned __tsan_mutex_try_lock = 1 << 4; +static const unsigned __tsan_mutex_try_lock = 1 << 4; // Denotes that a try lock operation has failed to acquire the mutex. -static const unsigned __tsan_mutex_try_lock_failed = 1 << 5; +static const unsigned __tsan_mutex_try_lock_failed = 1 << 5; // Denotes that the lock operation acquires multiple recursion levels. // Number of levels is passed in recursion parameter. // This is useful for annotation of e.g. Java builtin monitors, // for which wait operation releases all recursive acquisitions of the mutex. -static const unsigned __tsan_mutex_recursive_lock = 1 << 6; +static const unsigned __tsan_mutex_recursive_lock = 1 << 6; // Denotes that the unlock operation releases all recursion levels. // Number of released levels is returned and later must be passed to // the corresponding __tsan_mutex_post_lock annotation. @@ -75,20 +75,20 @@ static const unsigned __tsan_mutex_try_read_lock_failed = // Annotate creation of a mutex. // Supported flags: mutex creation flags. -void __tsan_mutex_create(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_create(void *addr, unsigned flags); // Annotate destruction of a mutex. // Supported flags: // - __tsan_mutex_linker_init // - __tsan_mutex_not_static -void __tsan_mutex_destroy(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_destroy(void *addr, unsigned flags); // Annotate start of lock operation. // Supported flags: // - __tsan_mutex_read_lock // - __tsan_mutex_try_lock // - all mutex creation flags -void __tsan_mutex_pre_lock(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_pre_lock(void *addr, unsigned flags); // Annotate end of lock operation. // Supported flags: @@ -97,23 +97,24 @@ void __tsan_mutex_pre_lock(void *addr, unsigned flags); // - __tsan_mutex_try_lock_failed // - __tsan_mutex_recursive_lock // - all mutex creation flags -void __tsan_mutex_post_lock(void *addr, unsigned flags, int recursion); +void SANITIZER_CDECL __tsan_mutex_post_lock(void *addr, unsigned flags, + int recursion); // Annotate start of unlock operation. // Supported flags: // - __tsan_mutex_read_lock // - __tsan_mutex_recursive_unlock -int __tsan_mutex_pre_unlock(void *addr, unsigned flags); +int SANITIZER_CDECL __tsan_mutex_pre_unlock(void *addr, unsigned flags); // Annotate end of unlock operation. // Supported flags: // - __tsan_mutex_read_lock (must match __tsan_mutex_pre_unlock) -void __tsan_mutex_post_unlock(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_post_unlock(void *addr, unsigned flags); // Annotate start/end of notify/signal/broadcast operation. // Supported flags: none. -void __tsan_mutex_pre_signal(void *addr, unsigned flags); -void __tsan_mutex_post_signal(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_pre_signal(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_post_signal(void *addr, unsigned flags); // Annotate start/end of a region of code where lock/unlock/signal operation // diverts to do something else unrelated to the mutex. This can be used to @@ -123,8 +124,12 @@ void __tsan_mutex_post_signal(void *addr, unsigned flags); // __tsan_mutex_pre/post_lock, __tsan_mutex_pre/post_unlock, // __tsan_mutex_pre/post_signal regions. // Supported flags: none. -void __tsan_mutex_pre_divert(void *addr, unsigned flags); -void __tsan_mutex_post_divert(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_pre_divert(void *addr, unsigned flags); +void SANITIZER_CDECL __tsan_mutex_post_divert(void *addr, unsigned flags); + +// Check that the current thread does not hold any mutexes, +// report a bug report otherwise. +void SANITIZER_CDECL __tsan_check_no_mutexes_held(); // External race detection API. // Can be used by non-instrumented libraries to detect when their objects are @@ -136,11 +141,14 @@ void __tsan_mutex_post_divert(void *addr, unsigned flags); // - __tsan_external_register_tag registers a 'tag' with the specified name, // which is later used in read/write annotations to denote the object type // - __tsan_external_assign_tag can optionally mark a heap object with a tag -void *__tsan_external_register_tag(const char *object_type); -void __tsan_external_register_header(void *tag, const char *header); -void __tsan_external_assign_tag(void *addr, void *tag); -void __tsan_external_read(void *addr, void *caller_pc, void *tag); -void __tsan_external_write(void *addr, void *caller_pc, void *tag); +void *SANITIZER_CDECL __tsan_external_register_tag(const char *object_type); +void SANITIZER_CDECL __tsan_external_register_header(void *tag, + const char *header); +void SANITIZER_CDECL __tsan_external_assign_tag(void *addr, void *tag); +void SANITIZER_CDECL __tsan_external_read(void *addr, void *caller_pc, + void *tag); +void SANITIZER_CDECL __tsan_external_write(void *addr, void *caller_pc, + void *tag); // Fiber switching API. // - TSAN context for fiber can be created by __tsan_create_fiber @@ -150,36 +158,159 @@ void __tsan_external_write(void *addr, void *caller_pc, void *tag); // - __tsan_switch_to_fiber should be called immediately before switch // to fiber, such as call of swapcontext. // - Fiber name can be set by __tsan_set_fiber_name. -void *__tsan_get_current_fiber(void); -void *__tsan_create_fiber(unsigned flags); -void __tsan_destroy_fiber(void *fiber); -void __tsan_switch_to_fiber(void *fiber, unsigned flags); -void __tsan_set_fiber_name(void *fiber, const char *name); +void *SANITIZER_CDECL __tsan_get_current_fiber(void); +void *SANITIZER_CDECL __tsan_create_fiber(unsigned flags); +void SANITIZER_CDECL __tsan_destroy_fiber(void *fiber); +void SANITIZER_CDECL __tsan_switch_to_fiber(void *fiber, unsigned flags); +void SANITIZER_CDECL __tsan_set_fiber_name(void *fiber, const char *name); // Flags for __tsan_switch_to_fiber: // Do not establish a happens-before relation between fibers static const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0; // User-provided callback invoked on TSan initialization. -void __tsan_on_initialize(); +void SANITIZER_CDECL __tsan_on_initialize(); // User-provided callback invoked on TSan shutdown. // `failed` - Nonzero if TSan did detect issues, zero otherwise. // Return `0` if TSan should exit as if no issues were detected. Return nonzero // if TSan should exit as if issues were detected. -int __tsan_on_finalize(int failed); +int SANITIZER_CDECL __tsan_on_finalize(int failed); // Release TSan internal memory in a best-effort manner. -void __tsan_flush_memory(); +void SANITIZER_CDECL __tsan_flush_memory(); // User-provided default TSAN options. -const char* __tsan_default_options(void); +const char *SANITIZER_CDECL __tsan_default_options(void); // User-provided default TSAN suppressions. -const char* __tsan_default_suppressions(void); +const char *SANITIZER_CDECL __tsan_default_suppressions(void); + +/// Returns a report's description. +/// +/// Returns a report's description (issue type), number of duplicate issues +/// found, counts of array data (stack traces, memory operations, locations, +/// mutexes, threads, unique thread IDs) and a stack trace of a <c>sleep()</c> +/// call (if one was involved in the issue). +/// +/// \param report Opaque pointer to the current report. +/// \param[out] description Report type description. +/// \param[out] count Count of duplicate issues. +/// \param[out] stack_count Count of stack traces. +/// \param[out] mop_count Count of memory operations. +/// \param[out] loc_count Count of locations. +/// \param[out] mutex_count Count of mutexes. +/// \param[out] thread_count Count of threads. +/// \param[out] unique_tid_count Count of unique thread IDs. +/// \param sleep_trace A buffer to store the stack trace of a <c>sleep()</c> +/// call. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_data( + void *report, const char **description, int *count, int *stack_count, + int *mop_count, int *loc_count, int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, unsigned long trace_size); + +/// Returns information about stack traces included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's stacks. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_stack(void *report, unsigned long idx, + void **trace, + unsigned long trace_size); + +/// Returns information about memory operations included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's memory operations. +/// \param[out] tid Thread ID of the memory operation. +/// \param[out] addr Address of the memory operation. +/// \param[out] size Size of the memory operation. +/// \param[out] write Write flag of the memory operation. +/// \param[out] atomic Atomicity flag of the memory operation. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_mop(void *report, unsigned long idx, + int *tid, void **addr, int *size, + int *write, int *atomic, void **trace, + unsigned long trace_size); + +/// Returns information about locations included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's locations. +/// \param[out] type Type of the location. +/// \param[out] addr Address of the location. +/// \param[out] start Start of the location. +/// \param[out] size Size of the location. +/// \param[out] tid Thread ID of the location. +/// \param[out] fd File descriptor of the location. +/// \param[out] suppressable Suppressable flag. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_loc(void *report, unsigned long idx, + const char **type, void **addr, + void **start, unsigned long *size, + int *tid, int *fd, int *suppressable, + void **trace, + unsigned long trace_size); + +/// Returns information about mutexes included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's mutexes. +/// \param[out] mutex_id Id of the mutex. +/// \param[out] addr Address of the mutex. +/// \param[out] destroyed Destroyed mutex flag. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_mutex(void *report, unsigned long idx, + uint64_t *mutex_id, void **addr, + int *destroyed, void **trace, + unsigned long trace_size); + +/// Returns information about threads included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's threads. +/// \param[out] tid Thread ID of the thread. +/// \param[out] os_id Operating system's ID of the thread. +/// \param[out] running Running flag of the thread. +/// \param[out] name Name of the thread. +/// \param[out] parent_tid ID of the parent thread. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_thread(void *report, unsigned long idx, + int *tid, uint64_t *os_id, + int *running, const char **name, + int *parent_tid, void **trace, + unsigned long trace_size); + +/// Returns information about unique thread IDs included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's unique thread IDs. +/// \param[out] tid Unique thread ID of the report. +/// \returns Returns 1 if successful, 0 if not. +int SANITIZER_CDECL __tsan_get_report_unique_tid(void *report, + unsigned long idx, int *tid); + +/// Returns the current report. +/// +/// If TSan is currently reporting a detected issue on the current thread, +/// returns an opaque pointer to the current report. Otherwise returns NULL. +/// \returns An opaque pointer to the current report. Otherwise returns NULL. +void *SANITIZER_CDECL __tsan_get_current_report(); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_TSAN_INTERFACE_H +#endif // SANITIZER_TSAN_INTERFACE_H diff --git a/libsanitizer/include/sanitizer/tsan_interface_atomic.h b/libsanitizer/include/sanitizer/tsan_interface_atomic.h index 5e41e2256c3..de3a1c39360 100644 --- a/libsanitizer/include/sanitizer/tsan_interface_atomic.h +++ b/libsanitizer/include/sanitizer/tsan_interface_atomic.h @@ -13,6 +13,8 @@ #ifndef TSAN_INTERFACE_ATOMIC_H #define TSAN_INTERFACE_ATOMIC_H +#include <sanitizer/common_interface_defs.h> + #ifdef __cplusplus extern "C" { #endif @@ -21,12 +23,12 @@ typedef char __tsan_atomic8; typedef short __tsan_atomic16; typedef int __tsan_atomic32; typedef long __tsan_atomic64; -#if defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302) +#if defined(__SIZEOF_INT128__) || \ + (__clang_major__ * 100 + __clang_minor__ >= 302) __extension__ typedef __int128 __tsan_atomic128; -# define __TSAN_HAS_INT128 1 +#define __TSAN_HAS_INT128 1 #else -# define __TSAN_HAS_INT128 0 +#define __TSAN_HAS_INT128 0 #endif // Part of ABI, do not change. @@ -40,182 +42,187 @@ typedef enum { __tsan_memory_order_seq_cst } __tsan_memory_order; -__tsan_atomic8 __tsan_atomic8_load(const volatile __tsan_atomic8 *a, - __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_load(const volatile __tsan_atomic16 *a, - __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_load(const volatile __tsan_atomic32 *a, - __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_load(const volatile __tsan_atomic64 *a, - __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL +__tsan_atomic8_load(const volatile __tsan_atomic8 *a, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL +__tsan_atomic16_load(const volatile __tsan_atomic16 *a, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL +__tsan_atomic32_load(const volatile __tsan_atomic32 *a, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL +__tsan_atomic64_load(const volatile __tsan_atomic64 *a, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_load(const volatile __tsan_atomic128 *a, - __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_load( + const volatile __tsan_atomic128 *a, __tsan_memory_order mo); #endif -void __tsan_atomic8_store(volatile __tsan_atomic8 *a, __tsan_atomic8 v, - __tsan_memory_order mo); -void __tsan_atomic16_store(volatile __tsan_atomic16 *a, __tsan_atomic16 v, - __tsan_memory_order mo); -void __tsan_atomic32_store(volatile __tsan_atomic32 *a, __tsan_atomic32 v, - __tsan_memory_order mo); -void __tsan_atomic64_store(volatile __tsan_atomic64 *a, __tsan_atomic64 v, - __tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic8_store(volatile __tsan_atomic8 *a, + __tsan_atomic8 v, + __tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic16_store(volatile __tsan_atomic16 *a, + __tsan_atomic16 v, + __tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic32_store(volatile __tsan_atomic32 *a, + __tsan_atomic32 v, + __tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic64_store(volatile __tsan_atomic64 *a, + __tsan_atomic64 v, + __tsan_memory_order mo); #if __TSAN_HAS_INT128 -void __tsan_atomic128_store(volatile __tsan_atomic128 *a, __tsan_atomic128 v, - __tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic128_store(volatile __tsan_atomic128 *a, + __tsan_atomic128 v, + __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_exchange(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_exchange(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_exchange(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_exchange(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_exchange( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_exchange( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_exchange( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_exchange( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_exchange(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_exchange( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_add(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_add(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_add(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_add(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_add( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_add( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_add( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_add( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_add(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_add( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_sub(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_sub(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_sub(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_sub(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_sub( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_sub( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_sub( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_sub( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_sub(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_sub( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_and(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_and(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_and(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_and(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_and( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_and( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_and( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_and( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_and(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_and( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_or(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_or(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_or(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_or(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_or( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_or( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_or( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_or( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_or(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_or( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_xor(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_xor(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_xor(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_xor(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_xor( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_xor( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_xor( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_xor( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_xor(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_xor( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -__tsan_atomic8 __tsan_atomic8_fetch_nand(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo); -__tsan_atomic16 __tsan_atomic16_fetch_nand(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo); -__tsan_atomic32 __tsan_atomic32_fetch_nand(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo); -__tsan_atomic64 __tsan_atomic64_fetch_nand(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo); +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_fetch_nand( + volatile __tsan_atomic8 *a, __tsan_atomic8 v, __tsan_memory_order mo); +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_fetch_nand( + volatile __tsan_atomic16 *a, __tsan_atomic16 v, __tsan_memory_order mo); +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_fetch_nand( + volatile __tsan_atomic32 *a, __tsan_atomic32 v, __tsan_memory_order mo); +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_fetch_nand( + volatile __tsan_atomic64 *a, __tsan_atomic64 v, __tsan_memory_order mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_fetch_nand(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo); +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_fetch_nand( + volatile __tsan_atomic128 *a, __tsan_atomic128 v, __tsan_memory_order mo); #endif -int __tsan_atomic8_compare_exchange_weak(volatile __tsan_atomic8 *a, - __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic16_compare_exchange_weak(volatile __tsan_atomic16 *a, - __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic32_compare_exchange_weak(volatile __tsan_atomic32 *a, - __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic64_compare_exchange_weak(volatile __tsan_atomic64 *a, - __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic8_compare_exchange_weak( + volatile __tsan_atomic8 *a, __tsan_atomic8 *c, __tsan_atomic8 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic16_compare_exchange_weak( + volatile __tsan_atomic16 *a, __tsan_atomic16 *c, __tsan_atomic16 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic32_compare_exchange_weak( + volatile __tsan_atomic32 *a, __tsan_atomic32 *c, __tsan_atomic32 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic64_compare_exchange_weak( + volatile __tsan_atomic64 *a, __tsan_atomic64 *c, __tsan_atomic64 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); #if __TSAN_HAS_INT128 -int __tsan_atomic128_compare_exchange_weak(volatile __tsan_atomic128 *a, - __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic128_compare_exchange_weak( + volatile __tsan_atomic128 *a, __tsan_atomic128 *c, __tsan_atomic128 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); #endif -int __tsan_atomic8_compare_exchange_strong(volatile __tsan_atomic8 *a, - __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic16_compare_exchange_strong(volatile __tsan_atomic16 *a, - __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic32_compare_exchange_strong(volatile __tsan_atomic32 *a, - __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); -int __tsan_atomic64_compare_exchange_strong(volatile __tsan_atomic64 *a, - __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic8_compare_exchange_strong( + volatile __tsan_atomic8 *a, __tsan_atomic8 *c, __tsan_atomic8 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic16_compare_exchange_strong( + volatile __tsan_atomic16 *a, __tsan_atomic16 *c, __tsan_atomic16 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic32_compare_exchange_strong( + volatile __tsan_atomic32 *a, __tsan_atomic32 *c, __tsan_atomic32 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic64_compare_exchange_strong( + volatile __tsan_atomic64 *a, __tsan_atomic64 *c, __tsan_atomic64 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); #if __TSAN_HAS_INT128 -int __tsan_atomic128_compare_exchange_strong(volatile __tsan_atomic128 *a, - __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo); +int SANITIZER_CDECL __tsan_atomic128_compare_exchange_strong( + volatile __tsan_atomic128 *a, __tsan_atomic128 *c, __tsan_atomic128 v, + __tsan_memory_order mo, __tsan_memory_order fail_mo); #endif -__tsan_atomic8 __tsan_atomic8_compare_exchange_val( +__tsan_atomic8 SANITIZER_CDECL __tsan_atomic8_compare_exchange_val( volatile __tsan_atomic8 *a, __tsan_atomic8 c, __tsan_atomic8 v, __tsan_memory_order mo, __tsan_memory_order fail_mo); -__tsan_atomic16 __tsan_atomic16_compare_exchange_val( +__tsan_atomic16 SANITIZER_CDECL __tsan_atomic16_compare_exchange_val( volatile __tsan_atomic16 *a, __tsan_atomic16 c, __tsan_atomic16 v, __tsan_memory_order mo, __tsan_memory_order fail_mo); -__tsan_atomic32 __tsan_atomic32_compare_exchange_val( +__tsan_atomic32 SANITIZER_CDECL __tsan_atomic32_compare_exchange_val( volatile __tsan_atomic32 *a, __tsan_atomic32 c, __tsan_atomic32 v, __tsan_memory_order mo, __tsan_memory_order fail_mo); -__tsan_atomic64 __tsan_atomic64_compare_exchange_val( +__tsan_atomic64 SANITIZER_CDECL __tsan_atomic64_compare_exchange_val( volatile __tsan_atomic64 *a, __tsan_atomic64 c, __tsan_atomic64 v, __tsan_memory_order mo, __tsan_memory_order fail_mo); #if __TSAN_HAS_INT128 -__tsan_atomic128 __tsan_atomic128_compare_exchange_val( +__tsan_atomic128 SANITIZER_CDECL __tsan_atomic128_compare_exchange_val( volatile __tsan_atomic128 *a, __tsan_atomic128 c, __tsan_atomic128 v, __tsan_memory_order mo, __tsan_memory_order fail_mo); #endif -void __tsan_atomic_thread_fence(__tsan_memory_order mo); -void __tsan_atomic_signal_fence(__tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic_thread_fence(__tsan_memory_order mo); +void SANITIZER_CDECL __tsan_atomic_signal_fence(__tsan_memory_order mo); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // TSAN_INTERFACE_ATOMIC_H +#endif // TSAN_INTERFACE_ATOMIC_H diff --git a/libsanitizer/include/sanitizer/ubsan_interface.h b/libsanitizer/include/sanitizer/ubsan_interface.h index 59fc6c3c184..435eb1ae332 100644 --- a/libsanitizer/include/sanitizer/ubsan_interface.h +++ b/libsanitizer/include/sanitizer/ubsan_interface.h @@ -23,10 +23,10 @@ extern "C" { /// <c>verbosity=1:halt_on_error=0</c>). /// /// \returns Default options string. -const char* __ubsan_default_options(void); +const char *SANITIZER_CDECL __ubsan_default_options(void); #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif -#endif // SANITIZER_UBSAN_INTERFACE_H +#endif // SANITIZER_UBSAN_INTERFACE_H diff --git a/libsanitizer/interception/interception.h b/libsanitizer/interception/interception.h index d97974ee907..069f73d276f 100644 --- a/libsanitizer/interception/interception.h +++ b/libsanitizer/interception/interception.h @@ -14,9 +14,10 @@ #ifndef INTERCEPTION_H #define INTERCEPTION_H +#include "sanitizer_common/sanitizer_asm.h" #include "sanitizer_common/sanitizer_internal_defs.h" -#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \ +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ !SANITIZER_SOLARIS # error "Interception doesn't work on this operating system." @@ -67,24 +68,50 @@ typedef __sanitizer::OFF64_T OFF64_T; // for more details). To intercept such functions you need to use the // INTERCEPTOR_WITH_SUFFIX(...) macro. -// How it works: -// To replace system functions on Linux we just need to declare functions -// with same names in our library and then obtain the real function pointers +// How it works on Linux +// --------------------- +// +// To replace system functions on Linux we just need to declare functions with +// the same names in our library and then obtain the real function pointers // using dlsym(). -// There is one complication. A user may also intercept some of the functions -// we intercept. To resolve this we declare our interceptors with __interceptor_ -// prefix, and then make actual interceptors weak aliases to __interceptor_ -// functions. // -// This is not so on Mac OS, where the two-level namespace makes -// our replacement functions invisible to other libraries. This may be overcomed -// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared -// libraries in Chromium were noticed when doing so. +// There is one complication: a user may also intercept some of the functions we +// intercept. To allow for up to 3 interceptors (including ours) of a given +// function "func", the interceptor implementation is in ___interceptor_func, +// which is aliased by a weak function __interceptor_func, which in turn is +// aliased (via a trampoline) by weak wrapper function "func". +// +// Most user interceptors should define a foreign interceptor as follows: +// +// - provide a non-weak function "func" that performs interception; +// - if __interceptor_func exists, call it to perform the real functionality; +// - if it does not exist, figure out the real function and call it instead. +// +// In rare cases, a foreign interceptor (of another dynamic analysis runtime) +// may be defined as follows (on supported architectures): +// +// - provide a non-weak function __interceptor_func that performs interception; +// - if ___interceptor_func exists, call it to perform the real functionality; +// - if it does not exist, figure out the real function and call it instead; +// - provide a weak function "func" that is an alias to __interceptor_func. +// +// With this protocol, sanitizer interceptors, foreign user interceptors, and +// foreign interceptors of other dynamic analysis runtimes, or any combination +// thereof, may co-exist simultaneously. +// +// How it works on Mac OS +// ---------------------- +// +// This is not so on Mac OS, where the two-level namespace makes our replacement +// functions invisible to other libraries. This may be overcomed using the +// DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared libraries in +// Chromium were noticed when doing so. +// // Instead we create a dylib containing a __DATA,__interpose section that // associates library functions with their wrappers. When this dylib is -// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all -// the calls to interposed functions done through stubs to the wrapper -// functions. +// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all the +// calls to interposed functions done through stubs to the wrapper functions. +// // As it's decided at compile time which functions are to be intercepted on Mac, // INTERCEPT_FUNCTION() is effectively a no-op on this system. @@ -100,53 +127,102 @@ struct interpose_substitution { // For a function foo() create a global pair of pointers { wrap_foo, foo } in // the __DATA,__interpose section. // As a result all the calls to foo() will be routed to wrap_foo() at runtime. -#define INTERPOSER(func_name) __attribute__((used)) \ +#define INTERPOSER(func_name) __attribute__((used)) \ const interpose_substitution substitution_##func_name[] \ __attribute__((section("__DATA, __interpose"))) = { \ - { reinterpret_cast<const uptr>(WRAP(func_name)), \ - reinterpret_cast<const uptr>(func_name) } \ + { reinterpret_cast<const uptr>(WRAP(func_name)), \ + reinterpret_cast<const uptr>(func_name) } \ } // For a function foo() and a wrapper function bar() create a global pair // of pointers { bar, foo } in the __DATA,__interpose section. // As a result all the calls to foo() will be routed to bar() at runtime. #define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \ -const interpose_substitution substitution_##func_name[] \ - __attribute__((section("__DATA, __interpose"))) = { \ - { reinterpret_cast<const uptr>(wrapper_name), \ - reinterpret_cast<const uptr>(func_name) } \ +const interpose_substitution substitution_##func_name[] \ + __attribute__((section("__DATA, __interpose"))) = { \ + { reinterpret_cast<const uptr>(wrapper_name), \ + reinterpret_cast<const uptr>(func_name) } \ } # define WRAP(x) wrap_##x -# define WRAPPER_NAME(x) "wrap_"#x +# define TRAMPOLINE(x) WRAP(x) # define INTERCEPTOR_ATTRIBUTE # define DECLARE_WRAPPER(ret_type, func, ...) #elif SANITIZER_WINDOWS # define WRAP(x) __asan_wrap_##x -# define WRAPPER_NAME(x) "__asan_wrap_"#x +# define TRAMPOLINE(x) WRAP(x) # define INTERCEPTOR_ATTRIBUTE __declspec(dllexport) -# define DECLARE_WRAPPER(ret_type, func, ...) \ +# define DECLARE_WRAPPER(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__); -# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ +# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); -#elif SANITIZER_FREEBSD || SANITIZER_NETBSD -# define WRAP(x) __interceptor_ ## x -# define WRAPPER_NAME(x) "__interceptor_" #x +#elif !SANITIZER_FUCHSIA // LINUX, FREEBSD, NETBSD, SOLARIS # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) +# if ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT +// Weak aliases of weak aliases do not work, therefore we need to set up a +// trampoline function. The function "func" is a weak alias to the trampoline +// (so that we may check if "func" was overridden), which calls the weak +// function __interceptor_func, which in turn aliases the actual interceptor +// implementation ___interceptor_func: +// +// [wrapper "func": weak] --(alias)--> [TRAMPOLINE(func)] +// | +// +--------(tail call)-------+ +// | +// v +// [__interceptor_func: weak] --(alias)--> [WRAP(func)] +// +// We use inline assembly to define most of this, because not all compilers +// support functions with the "naked" attribute with every architecture. +# define WRAP(x) ___interceptor_ ## x +# define TRAMPOLINE(x) __interceptor_trampoline_ ## x +# if SANITIZER_FREEBSD || SANITIZER_NETBSD // FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher // priority than weak ones so weak aliases won't work for indirect calls // in position-independent (-fPIC / -fPIE) mode. -# define DECLARE_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) \ - __attribute__((alias("__interceptor_" #func), visibility("default"))); -#elif !SANITIZER_FUCHSIA -# define WRAP(x) __interceptor_ ## x -# define WRAPPER_NAME(x) "__interceptor_" #x -# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) -# define DECLARE_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) \ - __attribute__((weak, alias("__interceptor_" #func), visibility("default"))); +# define __ASM_WEAK_WRAPPER(func) ".globl " #func "\n" +# else +# define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n" +# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD +// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h +# define DECLARE_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__); \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ + extern "C" ret_type __interceptor_##func(__VA_ARGS__) \ + INTERCEPTOR_ATTRIBUTE __attribute__((weak)) ALIAS(WRAP(func)); \ + asm( \ + ".text\n" \ + __ASM_WEAK_WRAPPER(func) \ + ".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n" \ + SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \ + SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \ + SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \ + SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \ + SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \ + ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ + ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ); +# else // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT +// Some architectures cannot implement efficient interceptor trampolines with +// just a plain jump due to complexities of resolving a preemptible symbol. In +// those cases, revert to just this scheme: +// +// [wrapper "func": weak] --(alias)--> [WRAP(func)] +// +# define WRAP(x) __interceptor_ ## x +# define TRAMPOLINE(x) WRAP(x) +# if SANITIZER_FREEBSD || SANITIZER_NETBSD +# define __ATTRIBUTE_WEAK_WRAPPER +# else +# define __ATTRIBUTE_WEAK_WRAPPER __attribute__((weak)) +# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD +# define DECLARE_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__) \ + INTERCEPTOR_ATTRIBUTE __ATTRIBUTE_WEAK_WRAPPER ALIAS(WRAP(func)); +# endif // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT #endif #if SANITIZER_FUCHSIA @@ -162,10 +238,10 @@ const interpose_substitution substitution_##func_name[] \ # define REAL(x) __interception::PTR_TO_REAL(x) # define FUNC_TYPE(x) x##_type -# define DECLARE_REAL(ret_type, func, ...) \ +# define DECLARE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ - namespace __interception { \ - extern FUNC_TYPE(func) PTR_TO_REAL(func); \ + namespace __interception { \ + extern FUNC_TYPE(func) PTR_TO_REAL(func); \ } # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src) #else // SANITIZER_APPLE @@ -176,14 +252,16 @@ const interpose_substitution substitution_##func_name[] \ #endif // SANITIZER_APPLE #if !SANITIZER_FUCHSIA -# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ extern "C" ret_type WRAP(func)(__VA_ARGS__); // Declare an interceptor and its wrapper defined in a different translation // unit (ex. asm). -# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type WRAP(func)(__VA_ARGS__); \ - extern "C" ret_type func(__VA_ARGS__); +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + extern "C" ret_type func(__VA_ARGS__); #else # define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) # define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) @@ -215,12 +293,10 @@ const interpose_substitution substitution_##func_name[] \ #elif !SANITIZER_APPLE -#define INTERCEPTOR(ret_type, func, ...) \ - DEFINE_REAL(ret_type, func, __VA_ARGS__) \ - DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ - extern "C" \ - INTERCEPTOR_ATTRIBUTE \ - ret_type WRAP(func)(__VA_ARGS__) +#define INTERCEPTOR(ret_type, func, ...) \ + DEFINE_REAL(ret_type, func, __VA_ARGS__) \ + DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__) // We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now. #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \ @@ -228,10 +304,10 @@ const interpose_substitution substitution_##func_name[] \ #else // SANITIZER_APPLE -#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) suffix; \ - extern "C" ret_type WRAP(func)(__VA_ARGS__); \ - INTERPOSER(func); \ +#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__) suffix; \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + INTERPOSER(func); \ extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__) #define INTERCEPTOR(ret_type, func, ...) \ @@ -246,14 +322,12 @@ const interpose_substitution substitution_##func_name[] \ #endif #if SANITIZER_WINDOWS -# define INTERCEPTOR_WINAPI(ret_type, func, ...) \ +# define INTERCEPTOR_WINAPI(ret_type, func, ...) \ typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \ - namespace __interception { \ - FUNC_TYPE(func) PTR_TO_REAL(func); \ - } \ - extern "C" \ - INTERCEPTOR_ATTRIBUTE \ - ret_type __stdcall WRAP(func)(__VA_ARGS__) + namespace __interception { \ + FUNC_TYPE(func) PTR_TO_REAL(func); \ + } \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type __stdcall WRAP(func)(__VA_ARGS__) #endif // ISO C++ forbids casting between pointer-to-function and pointer-to-object, diff --git a/libsanitizer/interception/interception_linux.cpp b/libsanitizer/interception/interception_linux.cpp index 5111a87f0a6..ef8136eb4fc 100644 --- a/libsanitizer/interception/interception_linux.cpp +++ b/libsanitizer/interception/interception_linux.cpp @@ -33,7 +33,7 @@ static int StrCmp(const char *s1, const char *s2) { } #endif -static void *GetFuncAddr(const char *name, uptr wrapper_addr) { +static void *GetFuncAddr(const char *name, uptr trampoline) { #if SANITIZER_NETBSD // FIXME: Find a better way to handle renames if (StrCmp(name, "sigaction")) @@ -50,17 +50,17 @@ static void *GetFuncAddr(const char *name, uptr wrapper_addr) { // In case `name' is not loaded, dlsym ends up finding the actual wrapper. // We don't want to intercept the wrapper and have it point to itself. - if ((uptr)addr == wrapper_addr) + if ((uptr)addr == trampoline) addr = nullptr; } return addr; } bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, - uptr wrapper) { - void *addr = GetFuncAddr(name, wrapper); + uptr trampoline) { + void *addr = GetFuncAddr(name, trampoline); *ptr_to_real = (uptr)addr; - return addr && (func == wrapper); + return addr && (func == trampoline); } // dlvsym is a GNU extension supported by some other platforms. @@ -70,12 +70,12 @@ static void *GetFuncAddr(const char *name, const char *ver) { } bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, - uptr func, uptr wrapper) { + uptr func, uptr trampoline) { void *addr = GetFuncAddr(name, ver); *ptr_to_real = (uptr)addr; - return addr && (func == wrapper); + return addr && (func == trampoline); } -#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD +# endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD } // namespace __interception diff --git a/libsanitizer/interception/interception_linux.h b/libsanitizer/interception/interception_linux.h index a08f8cb98c4..433a3d9bd7f 100644 --- a/libsanitizer/interception/interception_linux.h +++ b/libsanitizer/interception/interception_linux.h @@ -15,7 +15,7 @@ SANITIZER_SOLARIS #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) -# error "interception_linux.h should be included from interception library only" +# error interception_linux.h should be included from interception library only #endif #ifndef INTERCEPTION_LINUX_H @@ -23,26 +23,26 @@ namespace __interception { bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, - uptr wrapper); + uptr trampoline); bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, - uptr func, uptr wrapper); + uptr func, uptr trampoline); } // namespace __interception #define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ ::__interception::InterceptFunction( \ #func, \ - (::__interception::uptr *) & REAL(func), \ - (::__interception::uptr) & (func), \ - (::__interception::uptr) & WRAP(func)) + (::__interception::uptr *)&REAL(func), \ + (::__interception::uptr)&(func), \ + (::__interception::uptr)&TRAMPOLINE(func)) // dlvsym is a GNU extension supported by some other platforms. #if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ ::__interception::InterceptFunction( \ #func, symver, \ - (::__interception::uptr *) & REAL(func), \ - (::__interception::uptr) & (func), \ - (::__interception::uptr) & WRAP(func)) + (::__interception::uptr *)&REAL(func), \ + (::__interception::uptr)&(func), \ + (::__interception::uptr)&TRAMPOLINE(func)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) diff --git a/libsanitizer/interception/interception_win.cpp b/libsanitizer/interception/interception_win.cpp index faaa8ee1538..1b681ada37b 100644 --- a/libsanitizer/interception/interception_win.cpp +++ b/libsanitizer/interception/interception_win.cpp @@ -1,4 +1,4 @@ -//===-- interception_linux.cpp ----------------------------------*- C++ -*-===// +//===-- interception_win.cpp ------------------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -141,8 +141,29 @@ static const int kBranchLength = FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength); static const int kDirectBranchLength = kBranchLength + kAddressLength; +# if defined(_MSC_VER) +# define INTERCEPTION_FORMAT(f, a) +# else +# define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a))) +# endif + +static void (*ErrorReportCallback)(const char *format, ...) + INTERCEPTION_FORMAT(1, 2); + +void SetErrorReportCallback(void (*callback)(const char *format, ...)) { + ErrorReportCallback = callback; +} + +# define ReportError(...) \ + do { \ + if (ErrorReportCallback) \ + ErrorReportCallback(__VA_ARGS__); \ + } while (0) + static void InterceptionFailed() { - // Do we have a good way to abort with an error message here? + ReportError("interception_win: failed due to an unrecoverable error.\n"); + // This acts like an abort when no debugger is attached. According to an old + // comment, calling abort() leads to an infinite recursion in CheckFailed. __debugbreak(); } @@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) { } static void WriteJumpInstruction(uptr from, uptr target) { - if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) + if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) { + ReportError( + "interception_win: cannot write jmp further than 2GB away, from %p to " + "%p.\n", + (void *)from, (void *)target); InterceptionFailed(); + } ptrdiff_t offset = target - from - kJumpInstructionLength; *(u8*)from = 0xE9; *(u32*)(from + 1) = offset; @@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) { int offset = indirect_target - from - kIndirectJumpInstructionLength; if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength, indirect_target)) { + ReportError( + "interception_win: cannot write indirect jmp with target further than " + "2GB away, from %p to %p.\n", + (void *)from, (void *)indirect_target); InterceptionFailed(); } *(u16*)from = 0x25FF; @@ -427,6 +457,11 @@ static const u8 kPrologueWithShortJump2[] = { // Returns 0 on error. static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { +#if SANITIZER_ARM64 + // An ARM64 instruction is 4 bytes long. + return 4; +#endif + #if SANITIZER_WINDOWS64 if (memcmp((u8*)address, kPrologueWithShortJump1, sizeof(kPrologueWithShortJump1)) == 0 || @@ -492,6 +527,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xFF8B: // 8B FF : mov edi, edi case 0xEC8B: // 8B EC : mov ebp, esp case 0xc889: // 89 C8 : mov eax, ecx + case 0xE589: // 89 E5 : mov ebp, esp case 0xC18B: // 8B C1 : mov eax, ecx case 0xC033: // 33 C0 : xor eax, eax case 0xC933: // 33 C9 : xor ecx, ecx @@ -588,7 +624,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { // mov rax, QWORD PTR [rip + XXXXXXXX] case 0x25ff48: // 48 ff 25 XX XX XX XX : // rex.W jmp QWORD PTR [rip + XXXXXXXX] - + case 0x158D4C: // 4c 8d 15 XX XX XX XX : lea r10, [rip + XX] // Instructions having offset relative to 'rip' need offset adjustment. if (rel_offset) *rel_offset = 3; @@ -641,6 +677,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX] case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX] case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX] + case 0x245C8B: // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX] + case 0x246C8B: // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX] case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX] case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX] return 4; @@ -652,12 +690,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { } #endif - // Unknown instruction! - // FIXME: Unknown instruction failures might happen when we add a new - // interceptor or a new compiler version. In either case, they should result - // in visible and readable error messages. However, merely calling abort() - // leads to an infinite recursion in CheckFailed. - InterceptionFailed(); + // Unknown instruction! This might happen when we add a new interceptor, use + // a new compiler version, or if Windows changed how some functions are + // compiled. In either case, we print the address and 8 bytes of instructions + // to notify the user about the error and to help identify the unknown + // instruction. Don't treat this as a fatal error, though we can break the + // debugger if one has been attached. + u8 *bytes = (u8 *)address; + ReportError( + "interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x " + "%02x %02x %02x\n", + (void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7]); + if (::IsDebuggerPresent()) + __debugbreak(); return 0; } @@ -678,16 +724,24 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) { while (cursor != size) { size_t rel_offset = 0; size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset); - _memcpy((void*)(to + cursor), (void*)(from + cursor), + if (!instruction_size) + return false; + _memcpy((void *)(to + cursor), (void *)(from + cursor), (size_t)instruction_size); if (rel_offset) { - uptr delta = to - from; - uptr relocated_offset = *(u32*)(to + cursor + rel_offset) - delta; -#if SANITIZER_WINDOWS64 - if (relocated_offset + 0x80000000U >= 0xFFFFFFFFU) +# if SANITIZER_WINDOWS64 + // we want to make sure that the new relative offset still fits in 32-bits + // this will be untrue if relocated_offset \notin [-2**31, 2**31) + s64 delta = to - from; + s64 relocated_offset = *(s32 *)(to + cursor + rel_offset) - delta; + if (-0x8000'0000ll > relocated_offset || relocated_offset > 0x7FFF'FFFFll) return false; -#endif - *(u32*)(to + cursor + rel_offset) = relocated_offset; +# else + // on 32-bit, the relative offset will always be correct + s32 delta = to - from; + s32 relocated_offset = *(s32 *)(to + cursor + rel_offset) - delta; +# endif + *(s32 *)(to + cursor + rel_offset) = relocated_offset; } cursor += instruction_size; } @@ -895,6 +949,10 @@ static void **InterestingDLLsAvailable() { "msvcr120.dll", // VS2013 "vcruntime140.dll", // VS2015 "ucrtbase.dll", // Universal CRT +#if (defined(__MINGW32__) && defined(__i386__)) + "libc++.dll", // libc++ + "libunwind.dll", // libunwind +#endif // NTDLL should go last as it exports some functions that we should // override in the CRT [presumably only used internally]. "ntdll.dll", NULL}; diff --git a/libsanitizer/interception/interception_win.h b/libsanitizer/interception/interception_win.h index 4590013019e..f6eca82191c 100644 --- a/libsanitizer/interception/interception_win.h +++ b/libsanitizer/interception/interception_win.h @@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch, const char *function_name, uptr new_function, uptr *orig_old_func); +// Sets a callback to be used for reporting errors by interception_win. The +// callback will be called with printf-like arguments. Intended to be used with +// __sanitizer::Report. Pass nullptr to disable error reporting (default). +void SetErrorReportCallback(void (*callback)(const char *format, ...)); + #if !SANITIZER_WINDOWS64 // Exposed for unittests bool OverrideFunctionWithDetour( diff --git a/libsanitizer/lsan/lsan.cpp b/libsanitizer/lsan/lsan.cpp index 319f399e60f..6b223603c6a 100644 --- a/libsanitizer/lsan/lsan.cpp +++ b/libsanitizer/lsan/lsan.cpp @@ -97,7 +97,7 @@ extern "C" void __lsan_init() { ReplaceSystemMalloc(); InitTlsSize(); InitializeInterceptors(); - InitializeThreadRegistry(); + InitializeThreads(); InstallDeadlySignalHandlers(LsanOnDeadlySignal); InitializeMainThread(); InstallAtExitCheckLeaks(); diff --git a/libsanitizer/lsan/lsan_allocator.cpp b/libsanitizer/lsan/lsan_allocator.cpp index ee7facac6ea..12d579a9385 100644 --- a/libsanitizer/lsan/lsan_allocator.cpp +++ b/libsanitizer/lsan/lsan_allocator.cpp @@ -49,8 +49,11 @@ void InitializeAllocator() { max_malloc_size = kMaxAllowedMallocSize; } +void AllocatorThreadStart() { allocator.InitCache(GetAllocatorCache()); } + void AllocatorThreadFinish() { allocator.SwallowCache(GetAllocatorCache()); + allocator.DestroyCache(GetAllocatorCache()); } static ChunkMetadata *Metadata(const void *p) { @@ -65,12 +68,14 @@ static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { m->stack_trace_id = StackDepotPut(stack); m->requested_size = size; atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed); + RunMallocHooks(p, size); } static void RegisterDeallocation(void *p) { if (!p) return; ChunkMetadata *m = Metadata(p); CHECK(m); + RunFreeHooks(p); atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed); } @@ -104,7 +109,6 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment, if (cleared && allocator.FromPrimary(p)) memset(p, 0, size); RegisterAllocation(stack, p, size); - RunMallocHooks(p, size); return p; } @@ -119,7 +123,6 @@ static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) { } void Deallocate(void *p) { - RunFreeHooks(p); RegisterDeallocation(p); allocator.Deallocate(GetAllocatorCache(), p); } @@ -169,6 +172,10 @@ uptr GetMallocUsableSize(const void *p) { return m->requested_size; } +uptr GetMallocUsableSizeFast(const void *p) { + return Metadata(p)->requested_size; +} + int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, const StackTrace &stack) { if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { @@ -339,15 +346,6 @@ IgnoreObjectResult IgnoreObject(const void *p) { } } -void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) { - // This function can be used to treat memory reachable from `tctx` as live. - // This is useful for threads that have been created but not yet started. - - // This is currently a no-op because the LSan `pthread_create()` interceptor - // blocks until the child thread starts which keeps the thread's `arg` pointer - // live. -} - } // namespace __lsan using namespace __lsan; @@ -368,7 +366,7 @@ uptr __sanitizer_get_heap_size() { } SANITIZER_INTERFACE_ATTRIBUTE -uptr __sanitizer_get_free_bytes() { return 0; } +uptr __sanitizer_get_free_bytes() { return 1; } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_unmapped_bytes() { return 0; } @@ -377,7 +375,9 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } SANITIZER_INTERFACE_ATTRIBUTE -int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; } +int __sanitizer_get_ownership(const void *p) { + return GetMallocBegin(p) != nullptr; +} SANITIZER_INTERFACE_ATTRIBUTE const void * __sanitizer_get_allocated_begin(const void *p) { @@ -389,4 +389,15 @@ uptr __sanitizer_get_allocated_size(const void *p) { return GetMallocUsableSize(p); } +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = GetMallocUsableSizeFast(p); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); } + } // extern "C" diff --git a/libsanitizer/lsan/lsan_allocator.h b/libsanitizer/lsan/lsan_allocator.h index 10c1672ec5e..5eed0cbdb30 100644 --- a/libsanitizer/lsan/lsan_allocator.h +++ b/libsanitizer/lsan/lsan_allocator.h @@ -32,6 +32,7 @@ template<typename Callable> void ForEachChunk(const Callable &callback); void GetAllocatorCacheRange(uptr *begin, uptr *end); +void AllocatorThreadStart(); void AllocatorThreadFinish(); void InitializeAllocator(); @@ -67,20 +68,42 @@ using PrimaryAllocator = PrimaryAllocatorASVT<LocalAddressSpaceView>; #else # if SANITIZER_FUCHSIA || defined(__powerpc64__) const uptr kAllocatorSpace = ~(uptr)0; +# if SANITIZER_RISCV64 +// See the comments in compiler-rt/lib/asan/asan_allocator.h for why these +// values were chosen. +const uptr kAllocatorSize = UINT64_C(1) << 33; // 8GB +using LSanSizeClassMap = SizeClassMap</*kNumBits=*/2, + /*kMinSizeLog=*/5, + /*kMidSizeLog=*/8, + /*kMaxSizeLog=*/18, + /*kNumCachedHintT=*/8, + /*kMaxBytesCachedLog=*/10>; +static_assert(LSanSizeClassMap::kNumClassesRounded <= 32, + "32 size classes is the optimal number to ensure tests run " + "effieciently on Fuchsia."); +# else const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +using LSanSizeClassMap = DefaultSizeClassMap; +# endif +# elif SANITIZER_RISCV64 +const uptr kAllocatorSpace = ~(uptr)0; +const uptr kAllocatorSize = 0x2000000000ULL; // 128G. +using LSanSizeClassMap = DefaultSizeClassMap; # elif SANITIZER_APPLE const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +using LSanSizeClassMap = DefaultSizeClassMap; # else const uptr kAllocatorSpace = 0x500000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +using LSanSizeClassMap = DefaultSizeClassMap; # endif template <typename AddressSpaceViewTy> struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; static const uptr kSpaceSize = kAllocatorSize; static const uptr kMetadataSize = sizeof(ChunkMetadata); - typedef DefaultSizeClassMap SizeClassMap; + using SizeClassMap = LSanSizeClassMap; typedef NoOpMapUnmapCallback MapUnmapCallback; static const uptr kFlags = 0; using AddressSpaceView = AddressSpaceViewTy; diff --git a/libsanitizer/lsan/lsan_common.cpp b/libsanitizer/lsan/lsan_common.cpp index ae29e4a21fd..8b1af5b629f 100644 --- a/libsanitizer/lsan/lsan_common.cpp +++ b/libsanitizer/lsan/lsan_common.cpp @@ -34,15 +34,13 @@ # else # define OBJC_DATA_MASK 0x00007ffffffffff8UL # endif -// https://github.com/apple-oss-distributions/objc4/blob/8701d5672d3fd3cd817aeb84db1077aafe1a1604/runtime/objc-runtime-new.h#L139 -# define OBJC_FAST_IS_RW 0x8000000000000000UL # endif namespace __lsan { // This mutex is used to prevent races between DoLeakCheck and IgnoreObject, and // also to protect the global list of root regions. -Mutex global_mutex; +static Mutex global_mutex; Flags lsan_flags; @@ -173,13 +171,11 @@ static uptr GetCallerPC(const StackTrace &stack) { } # if SANITIZER_APPLE -// Objective-C class data pointers are stored with flags in the low bits, so -// they need to be transformed back into something that looks like a pointer. -static inline void *MaybeTransformPointer(void *p) { +// Several pointers in the Objective-C runtime (method cache and class_rw_t, +// for example) are tagged with additional bits we need to strip. +static inline void *TransformPointer(void *p) { uptr ptr = reinterpret_cast<uptr>(p); - if ((ptr & OBJC_FAST_IS_RW) == OBJC_FAST_IS_RW) - ptr &= OBJC_DATA_MASK; - return reinterpret_cast<void *>(ptr); + return reinterpret_cast<void *>(ptr & OBJC_DATA_MASK); } # endif @@ -241,12 +237,6 @@ static LeakSuppressionContext *GetSuppressionContext() { return suppression_ctx; } -static InternalMmapVectorNoCtor<RootRegion> root_regions; - -InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions() { - return &root_regions; -} - void InitCommonLsan() { if (common_flags()->detect_leaks) { // Initialization which can fail or print warnings should only be done if @@ -270,9 +260,14 @@ static inline bool MaybeUserPointer(uptr p) { if (p < kMinAddress) return false; # if defined(__x86_64__) - // TODO: add logic similar to ARM when Intel LAM is available. - // Accept only canonical form user-space addresses. - return ((p >> 47) == 0); + // TODO: support LAM48 and 5 level page tables. + // LAM_U57 mask format + // * top byte: 0x81 because the format is: [0] [6-bit tag] [0] + // * top-1 byte: 0xff because it should be 0 + // * top-2 byte: 0x80 because Linux uses 128 TB VMA ending at 0x7fffffffffff + constexpr uptr kLAM_U57Mask = 0x81ff80; + constexpr uptr kPointerMask = kLAM_U57Mask << 40; + return ((p & kPointerMask) == 0); # elif defined(__mips64) return ((p >> 40) == 0); # elif defined(__aarch64__) @@ -307,7 +302,7 @@ void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier, for (; pp + sizeof(void *) <= end; pp += alignment) { void *p = *reinterpret_cast<void **>(pp); # if SANITIZER_APPLE - p = MaybeTransformPointer(p); + p = TransformPointer(p); # endif if (!MaybeUserPointer(reinterpret_cast<uptr>(p))) continue; @@ -527,38 +522,52 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, # endif // SANITIZER_FUCHSIA -void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, - uptr region_begin, uptr region_end, bool is_readable) { - uptr intersection_begin = Max(root_region.begin, region_begin); - uptr intersection_end = Min(region_end, root_region.begin + root_region.size); - if (intersection_begin >= intersection_end) - return; - LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n", - (void *)root_region.begin, - (void *)(root_region.begin + root_region.size), - (void *)region_begin, (void *)region_end, - is_readable ? "readable" : "unreadable"); - if (is_readable) - ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT", - kReachable); +// A map that contains [region_begin, region_end) pairs. +using RootRegions = DenseMap<detail::DenseMapPair<uptr, uptr>, uptr>; + +static RootRegions &GetRootRegionsLocked() { + global_mutex.CheckLocked(); + static RootRegions *regions = nullptr; + alignas(RootRegions) static char placeholder[sizeof(RootRegions)]; + if (!regions) + regions = new (placeholder) RootRegions(); + return *regions; } -static void ProcessRootRegion(Frontier *frontier, - const RootRegion &root_region) { - MemoryMappingLayout proc_maps(/*cache_enabled*/ true); - MemoryMappedSegment segment; - while (proc_maps.Next(&segment)) { - ScanRootRegion(frontier, root_region, segment.start, segment.end, - segment.IsReadable()); +bool HasRootRegions() { return !GetRootRegionsLocked().empty(); } + +void ScanRootRegions(Frontier *frontier, + const InternalMmapVectorNoCtor<Region> &mapped_regions) { + if (!flags()->use_root_regions) + return; + + InternalMmapVector<Region> regions; + GetRootRegionsLocked().forEach([&](const auto &kv) { + regions.push_back({kv.first.first, kv.first.second}); + return true; + }); + + InternalMmapVector<Region> intersection; + Intersect(mapped_regions, regions, intersection); + + for (const Region &r : intersection) { + LOG_POINTERS("Root region intersects with mapped region at %p-%p\n", + (void *)r.begin, (void *)r.end); + ScanRangeForPointers(r.begin, r.end, frontier, "ROOT", kReachable); } } // Scans root regions for heap pointers. static void ProcessRootRegions(Frontier *frontier) { - if (!flags()->use_root_regions) + if (!flags()->use_root_regions || !HasRootRegions()) return; - for (uptr i = 0; i < root_regions.size(); i++) - ProcessRootRegion(frontier, root_regions[i]); + MemoryMappingLayout proc_maps(/*cache_enabled*/ true); + MemoryMappedSegment segment; + InternalMmapVector<Region> mapped_regions; + while (proc_maps.Next(&segment)) + if (segment.IsReadable()) + mapped_regions.push_back({segment.start, segment.end}); + ScanRootRegions(frontier, mapped_regions); } static void FloodFillTag(Frontier *frontier, ChunkTag tag) { @@ -941,8 +950,8 @@ void LeakReport::PrintSummary() { allocations += leaks_[i].hit_count; } InternalScopedString summary; - summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes, - allocations); + summary.AppendF("%zu byte(s) leaked in %zu allocation(s).", bytes, + allocations); ReportErrorSummary(summary.data()); } @@ -1013,36 +1022,37 @@ void __lsan_ignore_object(const void *p) { SANITIZER_INTERFACE_ATTRIBUTE void __lsan_register_root_region(const void *begin, uptr size) { #if CAN_SANITIZE_LEAKS - Lock l(&global_mutex); - RootRegion region = {reinterpret_cast<uptr>(begin), size}; - root_regions.push_back(region); VReport(1, "Registered root region at %p of size %zu\n", begin, size); + uptr b = reinterpret_cast<uptr>(begin); + uptr e = b + size; + CHECK_LT(b, e); + + Lock l(&global_mutex); + ++GetRootRegionsLocked()[{b, e}]; #endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE void __lsan_unregister_root_region(const void *begin, uptr size) { #if CAN_SANITIZE_LEAKS - Lock l(&global_mutex); - bool removed = false; - for (uptr i = 0; i < root_regions.size(); i++) { - RootRegion region = root_regions[i]; - if (region.begin == reinterpret_cast<uptr>(begin) && region.size == size) { - removed = true; - uptr last_index = root_regions.size() - 1; - root_regions[i] = root_regions[last_index]; - root_regions.pop_back(); - VReport(1, "Unregistered root region at %p of size %zu\n", begin, size); - break; + uptr b = reinterpret_cast<uptr>(begin); + uptr e = b + size; + CHECK_LT(b, e); + VReport(1, "Unregistered root region at %p of size %zu\n", begin, size); + + { + Lock l(&global_mutex); + if (auto *f = GetRootRegionsLocked().find({b, e})) { + if (--(f->second) == 0) + GetRootRegionsLocked().erase(f); + return; } } - if (!removed) { - Report( - "__lsan_unregister_root_region(): region at %p of size %zu has not " - "been registered.\n", - begin, size); - Die(); - } + Report( + "__lsan_unregister_root_region(): region at %p of size %zu has not " + "been registered.\n", + begin, size); + Die(); #endif // CAN_SANITIZE_LEAKS } diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h index a1f2d1a349b..d3e768363e9 100644 --- a/libsanitizer/lsan/lsan_common.h +++ b/libsanitizer/lsan/lsan_common.h @@ -18,6 +18,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_common/sanitizer_range.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_stoptheworld.h" #include "sanitizer_common/sanitizer_symbolizer.h" @@ -79,11 +80,6 @@ enum IgnoreObjectResult { kIgnoreObjectInvalid }; -struct Range { - uptr begin; - uptr end; -}; - //// -------------------------------------------------------------------------- //// Poisoning prototypes. //// -------------------------------------------------------------------------- @@ -96,8 +92,8 @@ bool WordIsPoisoned(uptr addr); //// -------------------------------------------------------------------------- // Wrappers for ThreadRegistry access. -void LockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; -void UnlockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; +void LockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; +void UnlockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; // If called from the main thread, updates the main thread's TID in the thread // registry. We need this to handle processes that fork() without a subsequent // exec(), which invalidates the recorded TID. To update it, we must call @@ -160,13 +156,13 @@ IgnoreObjectResult IgnoreObject(const void *p); struct ScopedStopTheWorldLock { ScopedStopTheWorldLock() { - LockThreadRegistry(); + LockThreads(); LockAllocator(); } ~ScopedStopTheWorldLock() { UnlockAllocator(); - UnlockThreadRegistry(); + UnlockThreads(); } ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete; @@ -239,11 +235,6 @@ void InitializePlatformSpecificModules(); void ProcessGlobalRegions(Frontier *frontier); void ProcessPlatformSpecificAllocations(Frontier *frontier); -struct RootRegion { - uptr begin; - uptr size; -}; - // LockStuffAndStopTheWorld can start to use Scan* calls to collect into // this Frontier vector before the StopTheWorldCallback actually runs. // This is used when the OS has a unified callback API for suspending @@ -256,9 +247,11 @@ struct CheckForLeaksParam { bool success = false; }; -InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions(); -void ScanRootRegion(Frontier *frontier, RootRegion const ®ion, - uptr region_begin, uptr region_end, bool is_readable); +using Region = Range; + +bool HasRootRegions(); +void ScanRootRegions(Frontier *frontier, + const InternalMmapVectorNoCtor<Region> ®ion); // Run stoptheworld while holding any platform-specific locks, as well as the // allocator and thread registry locks. void LockStuffAndStopTheWorld(StopTheWorldCallback callback, diff --git a/libsanitizer/lsan/lsan_common_fuchsia.cpp b/libsanitizer/lsan/lsan_common_fuchsia.cpp index bcad1c205fc..cb3fe1f859f 100644 --- a/libsanitizer/lsan/lsan_common_fuchsia.cpp +++ b/libsanitizer/lsan/lsan_common_fuchsia.cpp @@ -119,7 +119,8 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback, auto i = __sanitizer::InternalLowerBound(params->allocator_caches, begin); if (i < params->allocator_caches.size() && params->allocator_caches[i] >= begin && - end - params->allocator_caches[i] <= sizeof(AllocatorCache)) { + params->allocator_caches[i] <= end && + end - params->allocator_caches[i] >= sizeof(AllocatorCache)) { // Split the range in two and omit the allocator cache within. ScanRangeForPointers(begin, params->allocator_caches[i], ¶ms->argument->frontier, "TLS", kReachable); diff --git a/libsanitizer/lsan/lsan_common_mac.cpp b/libsanitizer/lsan/lsan_common_mac.cpp index 9ccf098a656..4e5198979b9 100644 --- a/libsanitizer/lsan/lsan_common_mac.cpp +++ b/libsanitizer/lsan/lsan_common_mac.cpp @@ -165,7 +165,8 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { vm_address_t address = 0; kern_return_t err = KERN_SUCCESS; - InternalMmapVectorNoCtor<RootRegion> const *root_regions = GetRootRegions(); + InternalMmapVector<Region> mapped_regions; + bool use_root_regions = flags()->use_root_regions && HasRootRegions(); RegionScanState scan_state; while (err == KERN_SUCCESS) { @@ -203,8 +204,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { // Recursing over the full memory map is very slow, break out // early if we don't need the full iteration. - if (scan_state.seen_regions == SeenRegion::All && - !(flags()->use_root_regions && root_regions->size() > 0)) { + if (scan_state.seen_regions == SeenRegion::All && !use_root_regions) { break; } @@ -215,15 +215,12 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { // // TODO(fjricci) - remove this once sanitizer_procmaps_mac has the same // behavior as sanitizer_procmaps_linux and traverses all memory regions - if (flags()->use_root_regions) { - for (uptr i = 0; i < root_regions->size(); i++) { - ScanRootRegion(frontier, (*root_regions)[i], address, end_address, - info.protection & kProtectionRead); - } - } + if (use_root_regions && (info.protection & kProtectionRead)) + mapped_regions.push_back({address, end_address}); address = end_address; } + ScanRootRegions(frontier, mapped_regions); } // On darwin, we can intercept _exit gracefully, and return a failing exit code diff --git a/libsanitizer/lsan/lsan_interceptors.cpp b/libsanitizer/lsan/lsan_interceptors.cpp index 3f8ef3fe48c..885f7ad5ddb 100644 --- a/libsanitizer/lsan/lsan_interceptors.cpp +++ b/libsanitizer/lsan/lsan_interceptors.cpp @@ -197,7 +197,7 @@ INTERCEPTOR(void*, pvalloc, uptr size) { #endif // SANITIZER_INTERCEPT_PVALLOC #if SANITIZER_INTERCEPT_CFREE -INTERCEPTOR(void, cfree, void *p) ALIAS(WRAPPER_NAME(free)); +INTERCEPTOR(void, cfree, void *p) ALIAS(WRAP(free)); #define LSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) #else #define LSAN_MAYBE_INTERCEPT_CFREE @@ -415,16 +415,10 @@ INTERCEPTOR(char *, strerror, int errnum) { #if SANITIZER_POSIX -struct ThreadParam { - void *(*callback)(void *arg); - void *param; - atomic_uintptr_t tid; -}; - -extern "C" void *__lsan_thread_start_func(void *arg) { - ThreadParam *p = (ThreadParam*)arg; - void* (*callback)(void *arg) = p->callback; - void *param = p->param; +template <bool Detached> +static void *ThreadStartFunc(void *arg) { + u32 parent_tid = (uptr)arg; + uptr tid = ThreadCreate(parent_tid, Detached); // Wait until the last iteration to maximize the chance that we are the last // destructor to run. #if !SANITIZER_NETBSD && !SANITIZER_FREEBSD @@ -433,55 +427,103 @@ extern "C" void *__lsan_thread_start_func(void *arg) { Report("LeakSanitizer: failed to set thread key.\n"); Die(); } -#endif - int tid = 0; - while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) - internal_sched_yield(); +# endif ThreadStart(tid, GetTid()); - atomic_store(&p->tid, 0, memory_order_release); - return callback(param); + auto self = GetThreadSelf(); + auto args = GetThreadArgRetval().GetArgs(self); + void *retval = (*args.routine)(args.arg_retval); + GetThreadArgRetval().Finish(self, retval); + return retval; } INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void *), void *param) { ENSURE_LSAN_INITED; EnsureMainThreadIDIsCorrect(); + + bool detached = [attr]() { + int d = 0; + return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); + }(); + __sanitizer_pthread_attr_t myattr; if (!attr) { pthread_attr_init(&myattr); attr = &myattr; } AdjustStackSize(attr); - int detached = 0; - pthread_attr_getdetachstate(attr, &detached); - ThreadParam p; - p.callback = callback; - p.param = param; - atomic_store(&p.tid, 0, memory_order_relaxed); - int res; + uptr this_tid = GetCurrentThreadId(); + int result; { // Ignore all allocations made by pthread_create: thread stack/TLS may be // stored by pthread for future reuse even after thread destruction, and // the linked list it's stored in doesn't even hold valid pointers to the // objects, the latter are calculated by obscure pointer arithmetic. ScopedInterceptorDisabler disabler; - res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, &p); - } - if (res == 0) { - int tid = ThreadCreate(GetCurrentThreadId(), IsStateDetached(detached)); - CHECK_NE(tid, kMainTid); - atomic_store(&p.tid, tid, memory_order_release); - while (atomic_load(&p.tid, memory_order_acquire) != 0) - internal_sched_yield(); + GetThreadArgRetval().Create(detached, {callback, param}, [&]() -> uptr { + result = REAL(pthread_create)( + th, attr, detached ? ThreadStartFunc<true> : ThreadStartFunc<false>, + (void *)this_tid); + return result ? 0 : *(uptr *)(th); + }); } if (attr == &myattr) pthread_attr_destroy(&myattr); - return res; + return result; } -INTERCEPTOR(int, pthread_join, void *t, void **arg) { - return REAL(pthread_join)(t, arg); +INTERCEPTOR(int, pthread_join, void *thread, void **retval) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_join)(thread, retval); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_detach, void *thread) { + int result; + GetThreadArgRetval().Detach((uptr)thread, [&]() { + result = REAL(pthread_detach)(thread); + return !result; + }); + return result; +} + +INTERCEPTOR(void, pthread_exit, void *retval) { + GetThreadArgRetval().Finish(GetThreadSelf(), retval); + REAL(pthread_exit)(retval); +} + +# if SANITIZER_INTERCEPT_TRYJOIN +INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_tryjoin_np)(thread, ret); + return !result; + }); + return result; +} +# define LSAN_MAYBE_INTERCEPT_TRYJOIN INTERCEPT_FUNCTION(pthread_tryjoin_np) +# else +# define LSAN_MAYBE_INTERCEPT_TRYJOIN +# endif // SANITIZER_INTERCEPT_TRYJOIN + +# if SANITIZER_INTERCEPT_TIMEDJOIN +INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, + const struct timespec *abstime) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_timedjoin_np)(thread, ret, abstime); + return !result; + }); + return result; } +# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN \ + INTERCEPT_FUNCTION(pthread_timedjoin_np) +# else +# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN +# endif // SANITIZER_INTERCEPT_TIMEDJOIN DEFINE_REAL_PTHREAD_FUNCTIONS @@ -491,6 +533,7 @@ INTERCEPTOR(void, _exit, int status) { } #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_LSAN_INITED #include "sanitizer_common/sanitizer_signal_interceptors.inc" #endif // SANITIZER_POSIX @@ -518,6 +561,10 @@ void InitializeInterceptors() { LSAN_MAYBE_INTERCEPT_MALLOPT; INTERCEPT_FUNCTION(pthread_create); INTERCEPT_FUNCTION(pthread_join); + INTERCEPT_FUNCTION(pthread_detach); + INTERCEPT_FUNCTION(pthread_exit); + LSAN_MAYBE_INTERCEPT_TIMEDJOIN; + LSAN_MAYBE_INTERCEPT_TRYJOIN; INTERCEPT_FUNCTION(_exit); LSAN_MAYBE_INTERCEPT__LWP_EXIT; diff --git a/libsanitizer/lsan/lsan_mac.cpp b/libsanitizer/lsan/lsan_mac.cpp index 2bcd0057f24..990954a8b68 100644 --- a/libsanitizer/lsan/lsan_mac.cpp +++ b/libsanitizer/lsan/lsan_mac.cpp @@ -80,7 +80,7 @@ extern "C" void lsan_dispatch_call_block_and_release(void *block) { VReport(2, "lsan_dispatch_call_block_and_release(): " "context: %p, pthread_self: %p\n", - block, pthread_self()); + block, (void*)pthread_self()); lsan_register_worker_thread(context->parent_tid); // Call the original dispatcher for the block. context->func(context->block); diff --git a/libsanitizer/lsan/lsan_thread.cpp b/libsanitizer/lsan/lsan_thread.cpp index 9da42f32e06..8aa3111eecf 100644 --- a/libsanitizer/lsan/lsan_thread.cpp +++ b/libsanitizer/lsan/lsan_thread.cpp @@ -24,6 +24,7 @@ namespace __lsan { static ThreadRegistry *thread_registry; +static ThreadArgRetval *thread_arg_retval; static Mutex mu_for_thread_context; static LowLevelAllocator allocator_for_thread_context; @@ -33,16 +34,26 @@ static ThreadContextBase *CreateThreadContext(u32 tid) { return new (allocator_for_thread_context) ThreadContext(tid); } -void InitializeThreadRegistry() { - static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)]; +void InitializeThreads() { + static ALIGNED(alignof( + ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)]; thread_registry = new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext); + + static ALIGNED(alignof(ThreadArgRetval)) char + thread_arg_retval_placeholder[sizeof(ThreadArgRetval)]; + thread_arg_retval = new (thread_arg_retval_placeholder) ThreadArgRetval(); } +ThreadArgRetval &GetThreadArgRetval() { return *thread_arg_retval; } + ThreadContextLsanBase::ThreadContextLsanBase(int tid) : ThreadContextBase(tid) {} -void ThreadContextLsanBase::OnStarted(void *arg) { SetCurrentThread(this); } +void ThreadContextLsanBase::OnStarted(void *arg) { + SetCurrentThread(this); + AllocatorThreadStart(); +} void ThreadContextLsanBase::OnFinished() { AllocatorThreadFinish(); @@ -72,9 +83,15 @@ void GetThreadExtraStackRangesLocked(tid_t os_id, InternalMmapVector<Range> *ranges) {} void GetThreadExtraStackRangesLocked(InternalMmapVector<Range> *ranges) {} -void LockThreadRegistry() { thread_registry->Lock(); } +void LockThreads() { + thread_registry->Lock(); + thread_arg_retval->Lock(); +} -void UnlockThreadRegistry() { thread_registry->Unlock(); } +void UnlockThreads() { + thread_arg_retval->Unlock(); + thread_registry->Unlock(); +} ThreadRegistry *GetLsanThreadRegistryLocked() { thread_registry->CheckLocked(); @@ -92,4 +109,8 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads) { threads); } +void GetAdditionalThreadContextPtrsLocked(InternalMmapVector<uptr> *ptrs) { + GetThreadArgRetval().GetAllPtrsLocked(ptrs); +} + } // namespace __lsan diff --git a/libsanitizer/lsan/lsan_thread.h b/libsanitizer/lsan/lsan_thread.h index 709a02915c2..222066ee93c 100644 --- a/libsanitizer/lsan/lsan_thread.h +++ b/libsanitizer/lsan/lsan_thread.h @@ -14,6 +14,7 @@ #ifndef LSAN_THREAD_H #define LSAN_THREAD_H +#include "sanitizer_common/sanitizer_thread_arg_retval.h" #include "sanitizer_common/sanitizer_thread_registry.h" namespace __lsan { @@ -43,10 +44,11 @@ class ThreadContextLsanBase : public ThreadContextBase { // This subclass of ThreadContextLsanBase is declared in an OS-specific header. class ThreadContext; -void InitializeThreadRegistry(); +void InitializeThreads(); void InitializeMainThread(); ThreadRegistry *GetLsanThreadRegistryLocked(); +ThreadArgRetval &GetThreadArgRetval(); u32 ThreadCreate(u32 tid, bool detached, void *arg = nullptr); void ThreadFinish(); diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am index cd8a9bf145d..02afe65620a 100644 --- a/libsanitizer/sanitizer_common/Makefile.am +++ b/libsanitizer/sanitizer_common/Makefile.am @@ -31,6 +31,7 @@ sanitizer_common_files = \ sanitizer_coverage_libcdep_new.cpp \ sanitizer_deadlock_detector1.cpp \ sanitizer_deadlock_detector2.cpp \ + sanitizer_dl.cpp \ sanitizer_errno.cpp \ sanitizer_file.cpp \ sanitizer_flags.cpp \ @@ -57,6 +58,7 @@ sanitizer_common_files = \ sanitizer_procmaps_linux.cpp \ sanitizer_procmaps_mac.cpp \ sanitizer_procmaps_solaris.cpp \ + sanitizer_range.cpp \ sanitizer_solaris.cpp \ sanitizer_stack_store.cpp \ sanitizer_stackdepot.cpp \ @@ -75,6 +77,7 @@ sanitizer_common_files = \ sanitizer_symbolizer_posix_libcdep.cpp \ sanitizer_symbolizer_win.cpp \ sanitizer_termination.cpp \ + sanitizer_thread_arg_retval.cpp \ sanitizer_thread_registry.cpp \ sanitizer_tls_get_addr.cpp \ sanitizer_unwind_linux_libcdep.cpp \ diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in index 6499036726b..881df60331b 100644 --- a/libsanitizer/sanitizer_common/Makefile.in +++ b/libsanitizer/sanitizer_common/Makefile.in @@ -125,9 +125,10 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \ sanitizer_common.lo sanitizer_common_libcdep.lo \ sanitizer_coverage_libcdep_new.lo \ sanitizer_deadlock_detector1.lo \ - sanitizer_deadlock_detector2.lo sanitizer_errno.lo \ - sanitizer_file.lo sanitizer_flags.lo sanitizer_flag_parser.lo \ - sanitizer_libc.lo sanitizer_libignore.lo sanitizer_linux.lo \ + sanitizer_deadlock_detector2.lo sanitizer_dl.lo \ + sanitizer_errno.lo sanitizer_file.lo sanitizer_flags.lo \ + sanitizer_flag_parser.lo sanitizer_libc.lo \ + sanitizer_libignore.lo sanitizer_linux.lo \ sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \ sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_mutex.lo \ sanitizer_netbsd.lo sanitizer_platform_limits_freebsd.lo \ @@ -138,20 +139,21 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \ sanitizer_posix_libcdep.lo sanitizer_printf.lo \ sanitizer_procmaps_bsd.lo sanitizer_procmaps_common.lo \ sanitizer_procmaps_linux.lo sanitizer_procmaps_mac.lo \ - sanitizer_procmaps_solaris.lo sanitizer_solaris.lo \ - sanitizer_stack_store.lo sanitizer_stackdepot.lo \ - sanitizer_stacktrace.lo sanitizer_stacktrace_libcdep.lo \ - sanitizer_stacktrace_sparc.lo sanitizer_symbolizer_mac.lo \ - sanitizer_symbolizer_report.lo sanitizer_stacktrace_printer.lo \ + sanitizer_procmaps_solaris.lo sanitizer_range.lo \ + sanitizer_solaris.lo sanitizer_stack_store.lo \ + sanitizer_stackdepot.lo sanitizer_stacktrace.lo \ + sanitizer_stacktrace_libcdep.lo sanitizer_stacktrace_sparc.lo \ + sanitizer_symbolizer_mac.lo sanitizer_symbolizer_report.lo \ + sanitizer_stacktrace_printer.lo \ sanitizer_stoptheworld_linux_libcdep.lo \ sanitizer_stoptheworld_mac.lo sanitizer_suppressions.lo \ sanitizer_symbolizer.lo sanitizer_symbolizer_libbacktrace.lo \ sanitizer_symbolizer_libcdep.lo \ sanitizer_symbolizer_posix_libcdep.lo \ sanitizer_symbolizer_win.lo sanitizer_termination.lo \ - sanitizer_thread_registry.lo sanitizer_tls_get_addr.lo \ - sanitizer_unwind_linux_libcdep.lo sanitizer_unwind_win.lo \ - sanitizer_win.lo + sanitizer_thread_arg_retval.lo sanitizer_thread_registry.lo \ + sanitizer_tls_get_addr.lo sanitizer_unwind_linux_libcdep.lo \ + sanitizer_unwind_win.lo sanitizer_win.lo am_libsanitizer_common_la_OBJECTS = $(am__objects_1) libsanitizer_common_la_OBJECTS = $(am_libsanitizer_common_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) @@ -390,6 +392,7 @@ sanitizer_common_files = \ sanitizer_coverage_libcdep_new.cpp \ sanitizer_deadlock_detector1.cpp \ sanitizer_deadlock_detector2.cpp \ + sanitizer_dl.cpp \ sanitizer_errno.cpp \ sanitizer_file.cpp \ sanitizer_flags.cpp \ @@ -416,6 +419,7 @@ sanitizer_common_files = \ sanitizer_procmaps_linux.cpp \ sanitizer_procmaps_mac.cpp \ sanitizer_procmaps_solaris.cpp \ + sanitizer_range.cpp \ sanitizer_solaris.cpp \ sanitizer_stack_store.cpp \ sanitizer_stackdepot.cpp \ @@ -434,6 +438,7 @@ sanitizer_common_files = \ sanitizer_symbolizer_posix_libcdep.cpp \ sanitizer_symbolizer_win.cpp \ sanitizer_termination.cpp \ + sanitizer_thread_arg_retval.cpp \ sanitizer_thread_registry.cpp \ sanitizer_tls_get_addr.cpp \ sanitizer_unwind_linux_libcdep.cpp \ @@ -546,6 +551,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_coverage_libcdep_new.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_deadlock_detector1.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_deadlock_detector2.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_dl.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_errno.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_file.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_flag_parser.Plo@am__quote@ @@ -572,6 +578,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_linux.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_solaris.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_range.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_solaris.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stack_store.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stackdepot.Plo@am__quote@ @@ -590,6 +597,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_report.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_symbolizer_win.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_termination.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_thread_arg_retval.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_thread_registry.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_tls_get_addr.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_unwind_linux_libcdep.Plo@am__quote@ diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp index 03392b61503..0513ae36fbc 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp @@ -138,14 +138,20 @@ void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS { // LowLevelAllocator constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; +constexpr uptr kMinNumPagesRounded = 16; +constexpr uptr kMinRoundedSize = 65536; static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; static LowLevelAllocateCallback low_level_alloc_callback; +static LowLevelAllocator Alloc; +LowLevelAllocator &GetGlobalLowLevelAllocator() { return Alloc; } + void *LowLevelAllocator::Allocate(uptr size) { // Align allocation size. size = RoundUpTo(size, low_level_alloc_min_alignment); if (allocated_end_ - allocated_current_ < (sptr)size) { - uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached()); + uptr size_to_allocate = RoundUpTo( + size, Min(GetPageSizeCached() * kMinNumPagesRounded, kMinRoundedSize)); allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__); allocated_end_ = allocated_current_ + size_to_allocate; if (low_level_alloc_callback) { diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.h b/libsanitizer/sanitizer_common/sanitizer_allocator.h index 76b936ff5ea..0b28f86d140 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator.h @@ -62,6 +62,13 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) { *rand_state = state; } +struct NoOpMapUnmapCallback { + void OnMap(uptr p, uptr size) const {} + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const {} + void OnUnmap(uptr p, uptr size) const {} +}; + #include "sanitizer_allocator_size_class_map.h" #include "sanitizer_allocator_stats.h" #include "sanitizer_allocator_primary64.h" diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h index b76d36dcf5a..49940d9b5d5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h @@ -29,9 +29,9 @@ class CombinedAllocator { LargeMmapAllocatorPtrArray, typename PrimaryAllocator::AddressSpaceView>; - void InitLinkerInitialized(s32 release_to_os_interval_ms) { - stats_.InitLinkerInitialized(); - primary_.Init(release_to_os_interval_ms); + void InitLinkerInitialized(s32 release_to_os_interval_ms, + uptr heap_start = 0) { + primary_.Init(release_to_os_interval_ms, heap_start); secondary_.InitLinkerInitialized(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h index 8f3b71eb6ce..de2b271fb0e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_interface.h @@ -25,6 +25,8 @@ SANITIZER_INTERFACE_ATTRIBUTE const void *__sanitizer_get_allocated_begin( const void *p); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_allocated_size(const void *p); +SANITIZER_INTERFACE_ATTRIBUTE uptr +__sanitizer_get_allocated_size_fast(const void *p); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes(); diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h index f2471efced6..52fe3fe3d15 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h @@ -353,7 +353,7 @@ class SizeClassAllocator32 { DCHECK_GT(max_count, 0); TransferBatch *b = nullptr; constexpr uptr kShuffleArraySize = 48; - uptr shuffle_array[kShuffleArraySize]; + UNINITIALIZED uptr shuffle_array[kShuffleArraySize]; uptr count = 0; for (uptr i = region; i < region + n_chunks * size; i += size) { shuffle_array[count++] = i; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h index 66ba71d325d..d77bc05b780 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h @@ -635,8 +635,9 @@ class SizeClassAllocator64 { return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg; } uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; } - // kRegionSize must be >= 2^32. - COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2))); + // kRegionSize should be able to satisfy the largest size class. + static_assert(kRegionSize >= SizeClassMap::kMaxSize, + "Region size exceed largest size"); // kRegionSize must be <= 2^36, see CompactPtrT. COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4))); // Call mmap for user memory with at least this size. diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h index 15764555560..0607819e7ef 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h @@ -82,7 +82,7 @@ class LargeMmapAllocator { InitLinkerInitialized(); } - void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { + void *Allocate(AllocatorStats *stat, const uptr size, uptr alignment) { CHECK(IsPowerOfTwo(alignment)); uptr map_size = RoundUpMapSize(size); if (alignment > page_size_) @@ -99,11 +99,11 @@ class LargeMmapAllocator { if (!map_beg) return nullptr; CHECK(IsAligned(map_beg, page_size_)); - MapUnmapCallback().OnMap(map_beg, map_size); uptr map_end = map_beg + map_size; uptr res = map_beg + page_size_; if (res & (alignment - 1)) // Align. res += alignment - (res & (alignment - 1)); + MapUnmapCallback().OnMapSecondary(map_beg, map_size, res, size); CHECK(IsAligned(res, alignment)); CHECK(IsAligned(res, page_size_)); CHECK_GE(res + size, map_beg); diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h b/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h index 6f14e3863c3..ae4dac9c8c9 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_stats.h @@ -25,19 +25,13 @@ typedef uptr AllocatorStatCounters[AllocatorStatCount]; // Per-thread stats, live in per-thread cache. class AllocatorStats { public: - void Init() { - internal_memset(this, 0, sizeof(*this)); - } - void InitLinkerInitialized() {} - + void Init() { internal_memset(this, 0, sizeof(*this)); } void Add(AllocatorStat i, uptr v) { - v += atomic_load(&stats_[i], memory_order_relaxed); - atomic_store(&stats_[i], v, memory_order_relaxed); + atomic_fetch_add(&stats_[i], v, memory_order_relaxed); } void Sub(AllocatorStat i, uptr v) { - v = atomic_load(&stats_[i], memory_order_relaxed) - v; - atomic_store(&stats_[i], v, memory_order_relaxed); + atomic_fetch_sub(&stats_[i], v, memory_order_relaxed); } void Set(AllocatorStat i, uptr v) { @@ -58,17 +52,13 @@ class AllocatorStats { // Global stats, used for aggregation and querying. class AllocatorGlobalStats : public AllocatorStats { public: - void InitLinkerInitialized() { - next_ = this; - prev_ = this; - } void Init() { internal_memset(this, 0, sizeof(*this)); - InitLinkerInitialized(); } void Register(AllocatorStats *s) { SpinMutexLock l(&mu_); + LazyInit(); s->next_ = next_; s->prev_ = this; next_->prev_ = s; @@ -87,7 +77,7 @@ class AllocatorGlobalStats : public AllocatorStats { internal_memset(s, 0, AllocatorStatCount * sizeof(uptr)); SpinMutexLock l(&mu_); const AllocatorStats *stats = this; - for (;;) { + for (; stats;) { for (int i = 0; i < AllocatorStatCount; i++) s[i] += stats->Get(AllocatorStat(i)); stats = stats->next_; @@ -100,6 +90,13 @@ class AllocatorGlobalStats : public AllocatorStats { } private: + void LazyInit() { + if (!next_) { + next_ = this; + prev_ = this; + } + } + mutable StaticSpinMutex mu_; }; diff --git a/libsanitizer/sanitizer_common/sanitizer_array_ref.h b/libsanitizer/sanitizer_common/sanitizer_array_ref.h new file mode 100644 index 00000000000..28d125383da --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_array_ref.h @@ -0,0 +1,123 @@ +//===-- sanitizer_array_ref.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ARRAY_REF_H +#define SANITIZER_ARRAY_REF_H + +#include "sanitizer_internal_defs.h" + +namespace __sanitizer { + +/// ArrayRef - Represent a constant reference to an array (0 or more elements +/// consecutively in memory), i.e. a start pointer and a length. It allows +/// various APIs to take consecutive elements easily and conveniently. +/// +/// This class does not own the underlying data, it is expected to be used in +/// situations where the data resides in some other buffer, whose lifetime +/// extends past that of the ArrayRef. For this reason, it is not in general +/// safe to store an ArrayRef. +/// +/// This is intended to be trivially copyable, so it should be passed by +/// value. +template <typename T> +class ArrayRef { + public: + constexpr ArrayRef() {} + constexpr ArrayRef(const T *begin, const T *end) : begin_(begin), end_(end) { + DCHECK(empty() || begin); + } + constexpr ArrayRef(const T *data, uptr length) + : ArrayRef(data, data + length) {} + template <uptr N> + constexpr ArrayRef(const T (&src)[N]) : ArrayRef(src, src + N) {} + template <typename C> + constexpr ArrayRef(const C &src) + : ArrayRef(src.data(), src.data() + src.size()) {} + ArrayRef(const T &one_elt) : ArrayRef(&one_elt, &one_elt + 1) {} + + const T *data() const { return empty() ? nullptr : begin_; } + + const T *begin() const { return begin_; } + const T *end() const { return end_; } + + bool empty() const { return begin_ == end_; } + + uptr size() const { return end_ - begin_; } + + /// equals - Check for element-wise equality. + bool equals(ArrayRef rhs) const { + if (size() != rhs.size()) + return false; + auto r = rhs.begin(); + for (auto &l : *this) { + if (!(l == *r)) + return false; + ++r; + } + return true; + } + + /// slice(n, m) - Chop off the first N elements of the array, and keep M + /// elements in the array. + ArrayRef<T> slice(uptr N, uptr M) const { + DCHECK_LE(N + M, size()); + return ArrayRef<T>(data() + N, M); + } + + /// slice(n) - Chop off the first N elements of the array. + ArrayRef<T> slice(uptr N) const { return slice(N, size() - N); } + + /// Drop the first \p N elements of the array. + ArrayRef<T> drop_front(uptr N = 1) const { + DCHECK_GE(size(), N); + return slice(N, size() - N); + } + + /// Drop the last \p N elements of the array. + ArrayRef<T> drop_back(uptr N = 1) const { + DCHECK_GE(size(), N); + return slice(0, size() - N); + } + + /// Return a copy of *this with only the first \p N elements. + ArrayRef<T> take_front(uptr N = 1) const { + if (N >= size()) + return *this; + return drop_back(size() - N); + } + + /// Return a copy of *this with only the last \p N elements. + ArrayRef<T> take_back(uptr N = 1) const { + if (N >= size()) + return *this; + return drop_front(size() - N); + } + + const T &operator[](uptr index) const { + DCHECK_LT(index, size()); + return begin_[index]; + } + + private: + const T *begin_ = nullptr; + const T *end_ = nullptr; +}; + +template <typename T> +inline bool operator==(ArrayRef<T> lhs, ArrayRef<T> rhs) { + return lhs.equals(rhs); +} + +template <typename T> +inline bool operator!=(ArrayRef<T> lhs, ArrayRef<T> rhs) { + return !(lhs == rhs); +} + +} // namespace __sanitizer + +#endif // SANITIZER_ARRAY_REF_H diff --git a/libsanitizer/sanitizer_common/sanitizer_asm.h b/libsanitizer/sanitizer_common/sanitizer_asm.h index 9ebba91da73..3c9bbdc9678 100644 --- a/libsanitizer/sanitizer_common/sanitizer_asm.h +++ b/libsanitizer/sanitizer_common/sanitizer_asm.h @@ -42,13 +42,57 @@ # define CFI_RESTORE(reg) #endif +#if defined(__x86_64__) || defined(__i386__) || defined(__sparc__) +# define ASM_TAIL_CALL jmp +#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__powerpc__) || defined(__loongarch_lp64) +# define ASM_TAIL_CALL b +#elif defined(__s390__) +# define ASM_TAIL_CALL jg +#elif defined(__riscv) +# define ASM_TAIL_CALL tail +#endif + +#if defined(__ELF__) && defined(__x86_64__) || defined(__i386__) || \ + defined(__riscv) +# define ASM_PREEMPTIBLE_SYM(sym) sym@plt +#else +# define ASM_PREEMPTIBLE_SYM(sym) sym +#endif + #if !defined(__APPLE__) # define ASM_HIDDEN(symbol) .hidden symbol # define ASM_TYPE_FUNCTION(symbol) .type symbol, %function # define ASM_SIZE(symbol) .size symbol, .-symbol # define ASM_SYMBOL(symbol) symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol -# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +# if defined(__i386__) || defined(__powerpc__) || defined(__s390__) || \ + defined(__sparc__) +// For details, see interception.h +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +# define ASM_TRAMPOLINE_ALIAS(symbol, name) \ + .weak symbol; \ + .set symbol, ASM_WRAPPER_NAME(name) +# define ASM_INTERCEPTOR_TRAMPOLINE(name) +# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 0 +# else // Architecture supports interceptor trampoline +// Keep trampoline implementation in sync with interception/interception.h +# define ASM_WRAPPER_NAME(symbol) ___interceptor_##symbol +# define ASM_TRAMPOLINE_ALIAS(symbol, name) \ + .weak symbol; \ + .set symbol, __interceptor_trampoline_##name +# define ASM_INTERCEPTOR_TRAMPOLINE(name) \ + .weak __interceptor_##name; \ + .set __interceptor_##name, ASM_WRAPPER_NAME(name); \ + .globl __interceptor_trampoline_##name; \ + ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \ + __interceptor_trampoline_##name: \ + CFI_STARTPROC; \ + ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \ + CFI_ENDPROC; \ + ASM_SIZE(__interceptor_trampoline_##name) +# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1 +# endif // Architecture supports interceptor trampoline #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cpp b/libsanitizer/sanitizer_common/sanitizer_common.cpp index 79b7748b8f6..5efdd864295 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common.cpp @@ -115,8 +115,9 @@ void ReportErrorSummary(const char *error_message, const char *alt_tool_name) { if (!common_flags()->print_summary) return; InternalScopedString buff; - buff.append("SUMMARY: %s: %s", - alt_tool_name ? alt_tool_name : SanitizerToolName, error_message); + buff.AppendF("SUMMARY: %s: %s", + alt_tool_name ? alt_tool_name : SanitizerToolName, + error_message); __sanitizer_report_error_summary(buff.data()); } diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h index 61d44020ed9..6b327a4aa16 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.h +++ b/libsanitizer/sanitizer_common/sanitizer_common.h @@ -117,6 +117,7 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, // unaccessible memory. bool MprotectNoAccess(uptr addr, uptr size); bool MprotectReadOnly(uptr addr, uptr size); +bool MprotectReadWrite(uptr addr, uptr size); void MprotectMallocZones(void *addr, int prot); @@ -207,6 +208,11 @@ void ParseUnixMemoryProfile(fill_profile_f cb, uptr *stats, char *smaps, // Simple low-level (mmap-based) allocator for internal use. Doesn't have // constructor, so all instances of LowLevelAllocator should be // linker initialized. +// +// NOTE: Users should instead use the singleton provided via +// `GetGlobalLowLevelAllocator()` rather than create a new one. This way, the +// number of mmap fragments can be reduced and use the same contiguous mmap +// provided by this singleton. class LowLevelAllocator { public: // Requires an external lock. @@ -223,6 +229,8 @@ typedef void (*LowLevelAllocateCallback)(uptr ptr, uptr size); // Passing NULL removes the callback. void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback); +LowLevelAllocator &GetGlobalLowLevelAllocator(); + // IO void CatastrophicErrorWrite(const char *buffer, uptr length); void RawWrite(const char *buffer); @@ -519,8 +527,8 @@ class InternalMmapVectorNoCtor { return data_[i]; } void push_back(const T &element) { - CHECK_LE(size_, capacity()); - if (size_ == capacity()) { + if (UNLIKELY(size_ >= capacity())) { + CHECK_EQ(size_, capacity()); uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1); Realloc(new_capacity); } @@ -580,7 +588,7 @@ class InternalMmapVectorNoCtor { } private: - void Realloc(uptr new_capacity) { + NOINLINE void Realloc(uptr new_capacity) { CHECK_GT(new_capacity, 0); CHECK_LE(size_, new_capacity); uptr new_capacity_bytes = @@ -635,7 +643,8 @@ class InternalScopedString { buffer_.resize(1); buffer_[0] = '\0'; } - void append(const char *format, ...) FORMAT(2, 3); + void Append(const char *str); + void AppendF(const char *format, ...) FORMAT(2, 3); const char *data() const { return buffer_.data(); } char *data() { return buffer_.data(); } @@ -796,7 +805,11 @@ inline const char *ModuleArchToString(ModuleArch arch) { return ""; } +#if SANITIZER_APPLE +const uptr kModuleUUIDSize = 16; +#else const uptr kModuleUUIDSize = 32; +#endif const uptr kMaxSegName = 16; // Represents a binary loaded into virtual memory (e.g. this can be an @@ -1079,20 +1092,6 @@ inline u32 GetNumberOfCPUsCached() { return NumberOfCPUsCached; } -template <typename T> -class ArrayRef { - public: - ArrayRef() {} - ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {} - - T *begin() { return begin_; } - T *end() { return end_; } - - private: - T *begin_ = nullptr; - T *end_ = nullptr; -}; - } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc index 490a8b12d8b..607ecae6808 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc @@ -26,25 +26,24 @@ // COMMON_INTERCEPTOR_SET_PTHREAD_NAME // COMMON_INTERCEPTOR_HANDLE_RECVMSG // COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED -// COMMON_INTERCEPTOR_MEMSET_IMPL -// COMMON_INTERCEPTOR_MEMMOVE_IMPL -// COMMON_INTERCEPTOR_MEMCPY_IMPL // COMMON_INTERCEPTOR_MMAP_IMPL +// COMMON_INTERCEPTOR_MUNMAP_IMPL // COMMON_INTERCEPTOR_COPY_STRING // COMMON_INTERCEPTOR_STRNDUP_IMPL // COMMON_INTERCEPTOR_STRERROR //===----------------------------------------------------------------------===// +#include <stdarg.h> + #include "interception/interception.h" #include "sanitizer_addrhashmap.h" +#include "sanitizer_dl.h" #include "sanitizer_errno.h" #include "sanitizer_placement_new.h" #include "sanitizer_platform_interceptors.h" #include "sanitizer_symbolizer.h" #include "sanitizer_tls_get_addr.h" -#include <stdarg.h> - #if SANITIZER_INTERCEPTOR_HOOKS #define CALL_WEAK_INTERCEPTOR_HOOK(f, ...) f(__VA_ARGS__); #define DECLARE_WEAK_INTERCEPTOR_HOOK(f, ...) \ @@ -198,15 +197,6 @@ extern const short *_tolower_tab_; #define wait4 __wait4_time64 #endif -// Platform-specific options. -#if SANITIZER_APPLE -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 -#elif SANITIZER_WINDOWS64 -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 -#else -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 -#endif // SANITIZER_APPLE - #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {} #endif @@ -302,53 +292,17 @@ extern const short *_tolower_tab_; COMMON_INTERCEPT_FUNCTION(fn) #endif -#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ - return internal_memset(dst, v, size); \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(dst, v, size); \ - } -#endif - -#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL -#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ - return internal_memmove(dst, src, size); \ - COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \ - if (common_flags()->intercept_intrin) { \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ - } \ - return REAL(memmove)(dst, src, size); \ - } -#endif - -#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL -#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \ - return internal_memmove(dst, src, size); \ - } \ - COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \ - if (common_flags()->intercept_intrin) { \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ - } \ - return REAL(memcpy)(dst, src, size); \ - } -#endif - #ifndef COMMON_INTERCEPTOR_MMAP_IMPL #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ off) \ { return REAL(mmap)(addr, sz, prot, flags, fd, off); } #endif +#ifndef COMMON_INTERCEPTOR_MUNMAP_IMPL +#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \ + { return REAL(munmap)(addr, sz); } +#endif + #ifndef COMMON_INTERCEPTOR_COPY_STRING #define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {} #endif @@ -492,11 +446,13 @@ INTERCEPTOR(char*, textdomain, const char *domainname) { #define INIT_TEXTDOMAIN #endif -#if SANITIZER_INTERCEPT_STRCMP +#if SANITIZER_INTERCEPT_STRCMP || SANITIZER_INTERCEPT_MEMCMP static inline int CharCmpX(unsigned char c1, unsigned char c2) { return (c1 == c2) ? 0 : (c1 < c2) ? -1 : 1; } +#endif +#if SANITIZER_INTERCEPT_STRCMP DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_strcmp, uptr called_pc, const char *s1, const char *s2, int result) @@ -841,57 +797,6 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) { #define INIT_STRPBRK #endif -#if SANITIZER_INTERCEPT_MEMSET -INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size); -} - -#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset) -#else -#define INIT_MEMSET -#endif - -#if SANITIZER_INTERCEPT_MEMMOVE -INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -} - -#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove) -#else -#define INIT_MEMMOVE -#endif - -#if SANITIZER_INTERCEPT_MEMCPY -INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { - // On OS X, calling internal_memcpy here will cause memory corruptions, - // because memcpy and memmove are actually aliases of the same - // implementation. We need to use internal_memmove here. - // N.B.: If we switch this to internal_ we'll have to use internal_memmove - // due to memcpy being an alias of memmove on OS X. - void *ctx; -#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); -#else - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -#endif -} - -#define INIT_MEMCPY \ - do { \ - if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \ - COMMON_INTERCEPT_FUNCTION(memcpy); \ - } else { \ - ASSIGN_REAL(memcpy, memmove); \ - } \ - CHECK(REAL(memcpy)); \ - } while (false) - -#else -#define INIT_MEMCPY -#endif - #if SANITIZER_INTERCEPT_MEMCMP DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, const void *s1, const void *s2, uptr n, @@ -1589,6 +1494,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap) INTERCEPTOR(int, __isoc99_vfscanf, void *stream, const char *format, va_list ap) VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap) + +INTERCEPTOR(int, __isoc23_vscanf, const char *format, va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vscanf, false, format, ap) + +INTERCEPTOR(int, __isoc23_vsscanf, const char *str, const char *format, + va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vsscanf, false, str, format, ap) + +INTERCEPTOR(int, __isoc23_vfscanf, void *stream, const char *format, va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vfscanf, false, stream, format, ap) #endif // SANITIZER_INTERCEPT_ISOC99_SCANF INTERCEPTOR(int, scanf, const char *format, ...) @@ -1609,6 +1524,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format) INTERCEPTOR(int, __isoc99_sscanf, const char *str, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) + +INTERCEPTOR(int, __isoc23_scanf, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_scanf, __isoc23_vscanf, format) + +INTERCEPTOR(int, __isoc23_fscanf, void *stream, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_fscanf, __isoc23_vfscanf, stream, format) + +INTERCEPTOR(int, __isoc23_sscanf, const char *str, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_sscanf, __isoc23_vsscanf, str, format) #endif #endif @@ -1632,7 +1556,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \ COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \ COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \ - COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); + COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_scanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_sscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_fscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vsscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vfscanf); #else #define INIT_ISOC99_SCANF #endif @@ -3416,7 +3346,8 @@ INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) { // its metadata. See // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent *res = REAL(readdir)(dirp); - if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res)); return res; } @@ -3431,7 +3362,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry, if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); if (*result) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result)); } return res; } @@ -3452,7 +3383,8 @@ INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) { // its metadata. See // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent64 *res = REAL(readdir64)(dirp); - if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res)); return res; } @@ -3467,7 +3399,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry, if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); if (*result) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result)); } return res; } @@ -3635,30 +3567,26 @@ UNUSED static inline void StrtolFixAndCheck(void *ctx, const char *nptr, (real_endptr - nptr) + 1 : 0); } - #if SANITIZER_INTERCEPT_STRTOIMAX -INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. +template <typename Fn> +static ALWAYS_INLINE auto StrtoimaxImpl(void *ctx, Fn real, const char *nptr, + char **endptr, int base) + -> decltype(real(nullptr, nullptr, 0)) { char *real_endptr; - INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base); + auto res = real(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); return res; } +INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(strtoimax), nptr, endptr, base); +} INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - char *real_endptr; - UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); - StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return res; + return StrtoimaxImpl(ctx, REAL(strtoumax), nptr, endptr, base); } #define INIT_STRTOIMAX \ @@ -3668,6 +3596,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { #define INIT_STRTOIMAX #endif +#if SANITIZER_INTERCEPT_STRTOIMAX && SANITIZER_GLIBC +INTERCEPTOR(INTMAX_T, __isoc23_strtoimax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoimax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(__isoc23_strtoimax), nptr, endptr, base); +} +INTERCEPTOR(UINTMAX_T, __isoc23_strtoumax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoumax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(__isoc23_strtoumax), nptr, endptr, base); +} + +# define INIT_STRTOIMAX_C23 \ + COMMON_INTERCEPT_FUNCTION(__isoc23_strtoimax); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_strtoumax); +#else +# define INIT_STRTOIMAX_C23 +#endif + #if SANITIZER_INTERCEPT_MBSTOWCS INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) { void *ctx; @@ -4039,7 +3986,7 @@ static THREADLOCAL scandir_compar_f scandir_compar; static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) { COMMON_INTERCEPTOR_UNPOISON_PARAM(1); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir)); return scandir_filter(dir); } @@ -4047,9 +3994,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a, const struct __sanitizer_dirent **b) { COMMON_INTERCEPTOR_UNPOISON_PARAM(2); COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a)); COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b)); return scandir_compar(a, b); } @@ -4073,7 +4020,7 @@ INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist, COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); for (int i = 0; i < res; ++i) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i], - (*namelist)[i]->d_reclen); + __sanitizer_dirsiz((*namelist)[i])); } return res; } @@ -4092,7 +4039,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar; static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) { COMMON_INTERCEPTOR_UNPOISON_PARAM(1); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir)); return scandir64_filter(dir); } @@ -4100,9 +4047,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a, const struct __sanitizer_dirent64 **b) { COMMON_INTERCEPTOR_UNPOISON_PARAM(2); COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a)); COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b)); return scandir64_compar(a, b); } @@ -4127,7 +4074,7 @@ INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist, COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); for (int i = 0; i < res; ++i) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i], - (*namelist)[i]->d_reclen); + __sanitizer_dirsiz((*namelist)[i])); } return res; } @@ -4404,12 +4351,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, INTERCEPTOR(int, backtrace, void **buffer, int size) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - int res = REAL(backtrace)(buffer, size); - if (res && buffer) + // 'buffer' might be freed memory, hence it is unsafe to directly call + // REAL(backtrace)(buffer, size). Instead, we use our own known-good + // scratch buffer. + void **scratch = (void**)InternalAlloc(sizeof(void*) * size); + int res = REAL(backtrace)(scratch, size); + if (res && buffer) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer)); + internal_memcpy(buffer, scratch, res * sizeof(*buffer)); + } + InternalFree(scratch); return res; } @@ -4418,9 +4369,8 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) { COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size); if (buffer && size) COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer)); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. + // The COMMON_INTERCEPTOR_READ_RANGE above ensures that 'buffer' is + // valid for reading. char **res = REAL(backtrace_symbols)(buffer, size); if (res && size) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res)); @@ -4453,7 +4403,7 @@ INTERCEPTOR(void, _exit, int status) { #if SANITIZER_INTERCEPT___LIBC_MUTEX INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate) -ALIAS(WRAPPER_NAME(pthread_setcancelstate)); +ALIAS(WRAP(pthread_setcancelstate)); #define INIT___LIBC_THR_SETCANCELSTATE \ COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate) @@ -5484,9 +5434,7 @@ INTERCEPTOR(void *, __tls_get_addr, void *arg) { // On PowerPC, we also need to intercept __tls_get_addr_opt, which has // mostly the same semantics as __tls_get_addr, but its presence enables // some optimizations in linker (which are safe to ignore here). -extern "C" __attribute__((alias("__interceptor___tls_get_addr"), - visibility("default"))) -void *__tls_get_addr_opt(void *arg); +INTERCEPTOR(void *, __tls_get_addr_opt, void *arg) ALIAS(WRAP(__tls_get_addr)); #endif #else // SANITIZER_S390 // On s390, we have to intercept two functions here: @@ -5520,21 +5468,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { #if SANITIZER_S390 && \ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET) -extern "C" uptr __tls_get_offset(void *arg); -extern "C" uptr __interceptor___tls_get_offset(void *arg); // We need a hidden symbol aliasing the above, so that we can jump // directly to it from the assembly below. -extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"), - visibility("hidden"))) -uptr __tls_get_addr_hidden(void *arg); +extern "C" __attribute__((visibility("hidden"))) uptr __tls_get_addr_hidden( + void *arg) ALIAS(WRAP(__tls_get_addr_internal)); +extern "C" uptr __tls_get_offset(void *arg); +extern "C" uptr TRAMPOLINE(__tls_get_offset)(void *arg); +extern "C" uptr WRAP(__tls_get_offset)(void *arg); // Now carefully intercept __tls_get_offset. asm( ".text\n" // The __intercept_ version has to exist, so that gen_dynamic_list.py // exports our symbol. ".weak __tls_get_offset\n" - ".type __tls_get_offset, @function\n" - "__tls_get_offset:\n" + ".set __tls_get_offset, __interceptor___tls_get_offset\n" ".global __interceptor___tls_get_offset\n" ".type __interceptor___tls_get_offset, @function\n" "__interceptor___tls_get_offset:\n" @@ -5790,105 +5737,6 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) { #define INIT_CAPGET #endif -#if SANITIZER_INTERCEPT_AEABI_MEM -INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -// Note the argument order. -INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -#define INIT_AEABI_MEM \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8); -#else -#define INIT_AEABI_MEM -#endif // SANITIZER_INTERCEPT_AEABI_MEM - -#if SANITIZER_INTERCEPT___BZERO -INTERCEPTOR(void *, __bzero, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} -#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); -#else -#define INIT___BZERO -#endif // SANITIZER_INTERCEPT___BZERO - -#if SANITIZER_INTERCEPT_BZERO -INTERCEPTOR(void *, bzero, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} -#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); -#else -#define INIT_BZERO -#endif // SANITIZER_INTERCEPT_BZERO - #if SANITIZER_INTERCEPT_FTIME INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { void *ctx; @@ -6460,7 +6308,36 @@ INTERCEPTOR(int, fclose, __sanitizer_FILE *fp) { INTERCEPTOR(void*, dlopen, const char *filename, int flag) { void *ctx; COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, dlopen, filename, flag); - if (filename) COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0); + + if (filename) { + COMMON_INTERCEPTOR_READ_STRING(ctx, filename, 0); + +# if !SANITIZER_DYNAMIC + // We care about a very specific use-case: dladdr on + // statically-linked ASan may return <main program> + // instead of the library. + // We therefore only take effect if the sanitizer is statically + // linked, and we don't bother canonicalizing paths because + // dladdr should return the same address both times (we assume + // the user did not canonicalize the result from dladdr). + if (common_flags()->test_only_replace_dlopen_main_program) { + VPrintf(1, "dlopen interceptor: filename: %s\n", filename); + + const char *SelfFName = DladdrSelfFName(); + VPrintf(1, "dlopen interceptor: DladdrSelfFName: %p %s\n", + (void *)SelfFName, SelfFName); + + if (internal_strcmp(SelfFName, filename) == 0) { + // It's possible they copied the string from dladdr, so + // we do a string comparison rather than pointer comparison. + VPrintf(1, "dlopen interceptor: replacing %s because it matches %s\n", + filename, SelfFName); + filename = (char *)0; // RTLD_DEFAULT + } + } +# endif // !SANITIZER_DYNAMIC + } + void *res = COMMON_INTERCEPTOR_DLOPEN(filename, flag); Symbolizer::GetOrInit()->InvalidateModuleList(); COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res); @@ -7173,6 +7050,7 @@ INTERCEPTOR(int, mprobe, void *ptr) { } #endif +#if SANITIZER_INTERCEPT_WCSLEN INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s); @@ -7191,6 +7069,9 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) { #define INIT_WCSLEN \ COMMON_INTERCEPT_FUNCTION(wcslen); \ COMMON_INTERCEPT_FUNCTION(wcsnlen); +#else +#define INIT_WCSLEN +#endif #if SANITIZER_INTERCEPT_WCSCAT INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) { @@ -7599,6 +7480,14 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd, COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off); } +INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return (int)internal_munmap(addr, sz); + COMMON_INTERCEPTOR_ENTER(ctx, munmap, addr, sz); + COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz); +} + INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) { void *ctx; if (common_flags()->detect_write_exec) @@ -7611,6 +7500,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) { } #define INIT_MMAP \ COMMON_INTERCEPT_FUNCTION(mmap); \ + COMMON_INTERCEPT_FUNCTION(munmap); \ COMMON_INTERCEPT_FUNCTION(mprotect); #else #define INIT_MMAP @@ -10355,14 +10245,33 @@ INTERCEPTOR(int, argp_parse, const struct argp *argp, int argc, char **argv, #define INIT_ARGP_PARSE #endif +#if SANITIZER_INTERCEPT_CPUSET_GETAFFINITY +INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T cpusetsize, __sanitizer_cpuset_t *mask) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cpuset_getaffinity, level, which, id, cpusetsize, mask); + int res = REAL(cpuset_getaffinity)(level, which, id, cpusetsize, mask); + if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize); + return res; +} +#define INIT_CPUSET_GETAFFINITY COMMON_INTERCEPT_FUNCTION(cpuset_getaffinity); +#else +#define INIT_CPUSET_GETAFFINITY +#endif + #include "sanitizer_common_interceptors_netbsd_compat.inc" +namespace __sanitizer { +void InitializeMemintrinsicInterceptors(); +} // namespace __sanitizer + static void InitializeCommonInterceptors() { #if SI_POSIX static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap(); #endif + __sanitizer::InitializeMemintrinsicInterceptors(); + INIT_MMAP; INIT_MMAP64; INIT_TEXTDOMAIN; @@ -10384,9 +10293,6 @@ static void InitializeCommonInterceptors() { INIT_STRPBRK; INIT_STRXFRM; INIT___STRXFRM_L; - INIT_MEMSET; - INIT_MEMMOVE; - INIT_MEMCPY; INIT_MEMCHR; INIT_MEMCMP; INIT_BCMP; @@ -10470,6 +10376,7 @@ static void InitializeCommonInterceptors() { INIT_GETCWD; INIT_GET_CURRENT_DIR_NAME; INIT_STRTOIMAX; + INIT_STRTOIMAX_C23; INIT_MBSTOWCS; INIT_MBSNRTOWCS; INIT_WCSTOMBS; @@ -10558,9 +10465,6 @@ static void InitializeCommonInterceptors() { INIT_GETIFADDRS; INIT_IF_INDEXTONAME; INIT_CAPGET; - INIT_AEABI_MEM; - INIT___BZERO; - INIT_BZERO; INIT_FTIME; INIT_XDR; INIT_XDRREC_LINUX; @@ -10673,6 +10577,7 @@ static void InitializeCommonInterceptors() { INIT___XUNAME; INIT_HEXDUMP; INIT_ARGP_PARSE; + INIT_CPUSET_GETAFFINITY; INIT___PRINTF_CHK; } diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc index 220abb89c3b..24e5dc0fb22 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_format.inc @@ -340,11 +340,19 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc, size = 0; } COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size); - // For %ms/%mc, write the allocated output buffer as well. + // For %mc/%mC/%ms/%m[/%mS, write the allocated output buffer as well. if (dir.allocate) { - char *buf = *(char **)argp; - if (buf) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1); + if (char *buf = *(char **)argp) { + if (dir.convSpecifier == 'c') + size = 1; + else if (dir.convSpecifier == 'C') + size = sizeof(wchar_t); + else if (dir.convSpecifier == 'S') + size = (internal_wcslen((wchar_t *)buf) + 1) * sizeof(wchar_t); + else // 's' or '[' + size = internal_strlen(buf) + 1; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size); + } } } } @@ -539,24 +547,25 @@ static void printf_common(void *ctx, const char *format, va_list aq) { continue; } else if (size == FSS_STRLEN) { if (void *argp = va_arg(aq, void *)) { + uptr len; if (dir.starredPrecision) { // FIXME: properly support starred precision for strings. - size = 0; + len = 0; } else if (dir.fieldPrecision > 0) { // Won't read more than "precision" symbols. - size = internal_strnlen((const char *)argp, dir.fieldPrecision); - if (size < dir.fieldPrecision) size++; + len = internal_strnlen((const char *)argp, dir.fieldPrecision); + if (len < (uptr)dir.fieldPrecision) + len++; } else { // Whole string will be accessed. - size = internal_strlen((const char *)argp) + 1; + len = internal_strlen((const char *)argp) + 1; } - COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size); + COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, len); } } else if (size == FSS_WCSLEN) { if (void *argp = va_arg(aq, void *)) { // FIXME: Properly support wide-character strings (via wcsrtombs). - size = 0; - COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, size); + COMMON_INTERCEPTOR_READ_RANGE(ctx, argp, 0); } } else { // Skip non-pointer args diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc new file mode 100644 index 00000000000..52e489d02cd --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc @@ -0,0 +1,244 @@ +//===-- sanitizer_common_interceptors_memintrinsics.inc ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Memintrinsic function interceptors for tools like AddressSanitizer, +// ThreadSanitizer, MemorySanitizer, etc. +// +// These interceptors are part of the common interceptors, but separated out so +// that implementations may add them, if necessary, to a separate source file +// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top. +// +// This file should be included into the tool's memintrinsic interceptor file, +// which has to define its own macros: +// COMMON_INTERCEPTOR_ENTER +// COMMON_INTERCEPTOR_READ_RANGE +// COMMON_INTERCEPTOR_WRITE_RANGE +// COMMON_INTERCEPTOR_MEMSET_IMPL +// COMMON_INTERCEPTOR_MEMMOVE_IMPL +// COMMON_INTERCEPTOR_MEMCPY_IMPL +// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED +//===----------------------------------------------------------------------===// + +#ifdef SANITIZER_REDEFINE_BUILTINS_H +#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file" +#endif + +#include "interception/interception.h" +#include "sanitizer_platform_interceptors.h" + +// Platform-specific options. +#if SANITIZER_APPLE +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 +#elif SANITIZER_WINDOWS64 +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 +#else +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 +#endif // SANITIZER_APPLE + +#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ + return internal_memset(dst, v, size); \ + COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ + if (common_flags()->intercept_intrin) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + return REAL(memset)(dst, v, size); \ + } +#endif + +#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ + return internal_memmove(dst, src, size); \ + COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \ + if (common_flags()->intercept_intrin) { \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ + } \ + return REAL(memmove)(dst, src, size); \ + } +#endif + +#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \ + return internal_memmove(dst, src, size); \ + } \ + COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \ + if (common_flags()->intercept_intrin) { \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ + } \ + return REAL(memcpy)(dst, src, size); \ + } +#endif + +#if SANITIZER_INTERCEPT_MEMSET +INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size); +} + +#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset) +#else +#define INIT_MEMSET +#endif + +#if SANITIZER_INTERCEPT_MEMMOVE +INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +} + +#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove) +#else +#define INIT_MEMMOVE +#endif + +#if SANITIZER_INTERCEPT_MEMCPY +INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { + // On OS X, calling internal_memcpy here will cause memory corruptions, + // because memcpy and memmove are actually aliases of the same + // implementation. We need to use internal_memmove here. + // N.B.: If we switch this to internal_ we'll have to use internal_memmove + // due to memcpy being an alias of memmove on OS X. + void *ctx; +#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); +#else + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +#endif +} + +#define INIT_MEMCPY \ + do { \ + if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \ + COMMON_INTERCEPT_FUNCTION(memcpy); \ + } else { \ + ASSIGN_REAL(memcpy, memmove); \ + } \ + CHECK(REAL(memcpy)); \ + } while (false) + +#else +#define INIT_MEMCPY +#endif + +#if SANITIZER_INTERCEPT_AEABI_MEM +INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +// Note the argument order. +INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +#define INIT_AEABI_MEM \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8); +#else +#define INIT_AEABI_MEM +#endif // SANITIZER_INTERCEPT_AEABI_MEM + +#if SANITIZER_INTERCEPT___BZERO +INTERCEPTOR(void *, __bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); +#else +#define INIT___BZERO +#endif // SANITIZER_INTERCEPT___BZERO + +#if SANITIZER_INTERCEPT_BZERO +INTERCEPTOR(void *, bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); +#else +#define INIT_BZERO +#endif // SANITIZER_INTERCEPT_BZERO + +namespace __sanitizer { +// This does not need to be called if InitializeCommonInterceptors() is called. +void InitializeMemintrinsicInterceptors() { + INIT_MEMSET; + INIT_MEMMOVE; + INIT_MEMCPY; + INIT_AEABI_MEM; + INIT___BZERO; + INIT_BZERO; +} +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S index 72e482754b6..cdfa6f1d7f5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S @@ -40,8 +40,8 @@ ASM_WRAPPER_NAME(vfork): ret ASM_SIZE(vfork) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) GNU_PROPERTY_BTI_PAC diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S index 780a9d46e26..87bb4838056 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_arm.inc.S @@ -43,7 +43,7 @@ ASM_WRAPPER_NAME(vfork): ASM_SIZE(vfork) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S index f60b05d157b..c633014e2da 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_i386.inc.S @@ -58,7 +58,7 @@ ASM_WRAPPER_NAME(vfork): ret ASM_SIZE(vfork) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S index 68782acb379..8429d57d669 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_loongarch64.inc.S @@ -51,7 +51,7 @@ ASM_WRAPPER_NAME(vfork): jr $ra ASM_SIZE(vfork) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S index b7ec27859b8..5b6ea6fe6c7 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_riscv64.inc.S @@ -50,7 +50,7 @@ ASM_WRAPPER_NAME(vfork): ret ASM_SIZE(vfork) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S index 8fd18ea67ff..5500f817aec 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_vfork_x86_64.inc.S @@ -34,9 +34,9 @@ ASM_WRAPPER_NAME(vfork): .L_exit: pop %rax ret -ASM_SIZE(vfork) +ASM_SIZE(ASM_WRAPPER_NAME(vfork)) -.weak vfork -.set vfork, ASM_WRAPPER_NAME(vfork) +ASM_INTERCEPTOR_TRAMPOLINE(vfork) +ASM_TRAMPOLINE_ALIAS(vfork, vfork) #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interface.inc b/libsanitizer/sanitizer_common/sanitizer_common_interface.inc index 01be600e33b..557207fe62a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interface.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interface.inc @@ -34,6 +34,7 @@ INTERFACE_FUNCTION(__sanitizer_symbolize_pc) // Allocator interface. INTERFACE_FUNCTION(__sanitizer_get_allocated_begin) INTERFACE_FUNCTION(__sanitizer_get_allocated_size) +INTERFACE_FUNCTION(__sanitizer_get_allocated_size_fast) INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes) INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size) INTERFACE_FUNCTION(__sanitizer_get_free_bytes) @@ -45,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator) INTERFACE_FUNCTION(__sanitizer_print_memory_profile) INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook) INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook) +// Memintrinsic functions. +INTERFACE_FUNCTION(__sanitizer_internal_memcpy) +INTERFACE_FUNCTION(__sanitizer_internal_memmove) +INTERFACE_FUNCTION(__sanitizer_internal_memset) diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc b/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc index a5259be9335..6b567edc97a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interface_posix.inc @@ -9,6 +9,7 @@ //===----------------------------------------------------------------------===// INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_code) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_data) +INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_frame) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_demangle) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_flush) INTERFACE_WEAK_FUNCTION(__sanitizer_symbolize_set_demangle) diff --git a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp index 8fd39856428..7b74bb1a7e0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp @@ -67,6 +67,8 @@ void *BackgroundThread(void *arg) { } else if (soft_rss_limit_mb >= current_rss_mb && reached_soft_rss_limit) { reached_soft_rss_limit = false; + Report("%s: soft rss limit unexhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, soft_rss_limit_mb, current_rss_mb); SetRssLimitExceeded(false); } } @@ -117,8 +119,10 @@ void MaybeStartBackgroudThread() {} #endif void WriteToSyslog(const char *msg) { + if (!msg) + return; InternalScopedString msg_copy; - msg_copy.append("%s", msg); + msg_copy.Append(msg); const char *p = msg_copy.data(); // Print one line at a time. diff --git a/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc b/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc index 3900bcf22b7..c10943b3e48 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_syscalls.inc @@ -2135,7 +2135,7 @@ PRE_SYSCALL(epoll_pwait2) const sanitizer_kernel_timespec *timeout, const kernel_sigset_t *sigmask, long sigsetsize) { if (timeout) - PRE_READ(timeout, sizeof(timeout)); + PRE_READ(timeout, sizeof(*timeout)); if (sigmask) PRE_READ(sigmask, sigsetsize); } diff --git a/libsanitizer/sanitizer_common/sanitizer_dl.cpp b/libsanitizer/sanitizer_common/sanitizer_dl.cpp new file mode 100644 index 00000000000..e957d529c2f --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_dl.cpp @@ -0,0 +1,37 @@ +//===-- sanitizer_dl.cpp --------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file has helper functions that depend on libc's dynamic loading +// introspection. +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_dl.h" + +#include "sanitizer_common/sanitizer_platform.h" + +#if SANITIZER_GLIBC +# include <dlfcn.h> +#endif + +namespace __sanitizer { +extern const char *SanitizerToolName; + +const char *DladdrSelfFName(void) { +#if SANITIZER_GLIBC + Dl_info info; + int ret = dladdr((void *)&SanitizerToolName, &info); + if (ret) { + return info.dli_fname; + } +#endif + + return nullptr; +} + +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_dl.h b/libsanitizer/sanitizer_common/sanitizer_dl.h new file mode 100644 index 00000000000..ecde0664eb0 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_dl.h @@ -0,0 +1,26 @@ +//===-- sanitizer_dl.h ----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file has helper functions that depend on libc's dynamic loading +// introspection. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_DL_H +#define SANITIZER_DL_H + +namespace __sanitizer { + +// Returns the path to the shared object or - in the case of statically linked +// sanitizers +// - the main program itself, that contains the sanitizer. +const char* DladdrSelfFName(void); + +} // namespace __sanitizer + +#endif // SANITIZER_DL_H diff --git a/libsanitizer/sanitizer_common/sanitizer_file.h b/libsanitizer/sanitizer_common/sanitizer_file.h index 9459c6b00ac..bef2c842d9f 100644 --- a/libsanitizer/sanitizer_common/sanitizer_file.h +++ b/libsanitizer/sanitizer_common/sanitizer_file.h @@ -84,7 +84,7 @@ bool IsPathSeparator(const char c); bool IsAbsolutePath(const char *path); // Returns true on success, false on failure. bool CreateDir(const char *pathname); -// Starts a subprocess and returs its pid. +// Starts a subprocess and returns its pid. // If *_fd parameters are not kInvalidFd their corresponding input/output // streams will be redirect to the file. The files will always be closed // in parent process even in case of an error. diff --git a/libsanitizer/sanitizer_common/sanitizer_flag_parser.cpp b/libsanitizer/sanitizer_common/sanitizer_flag_parser.cpp index c620da7f220..ca37df34858 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flag_parser.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_flag_parser.cpp @@ -19,8 +19,6 @@ namespace __sanitizer { -LowLevelAllocator FlagParser::Alloc; - class UnknownFlags { static const int kMaxUnknownFlags = 20; const char *unknown_flags_[kMaxUnknownFlags]; @@ -49,7 +47,7 @@ void ReportUnrecognizedFlags() { char *FlagParser::ll_strndup(const char *s, uptr n) { uptr len = internal_strnlen(s, n); - char *s2 = (char*)Alloc.Allocate(len + 1); + char *s2 = (char *)GetGlobalLowLevelAllocator().Allocate(len + 1); internal_memcpy(s2, s, len); s2[len] = 0; return s2; @@ -185,7 +183,8 @@ void FlagParser::RegisterHandler(const char *name, FlagHandlerBase *handler, } FlagParser::FlagParser() : n_flags_(0), buf_(nullptr), pos_(0) { - flags_ = (Flag *)Alloc.Allocate(sizeof(Flag) * kMaxFlags); + flags_ = + (Flag *)GetGlobalLowLevelAllocator().Allocate(sizeof(Flag) * kMaxFlags); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_flag_parser.h b/libsanitizer/sanitizer_common/sanitizer_flag_parser.h index ae49294dde9..dccdee4da2b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flag_parser.h +++ b/libsanitizer/sanitizer_common/sanitizer_flag_parser.h @@ -178,8 +178,6 @@ class FlagParser { bool ParseFile(const char *path, bool ignore_missing); void PrintFlagDescriptions(); - static LowLevelAllocator Alloc; - private: void fatal_error(const char *err); bool is_space(char c); @@ -193,7 +191,7 @@ class FlagParser { template <typename T> static void RegisterFlag(FlagParser *parser, const char *name, const char *desc, T *var) { - FlagHandler<T> *fh = new (FlagParser::Alloc) FlagHandler<T>(var); + FlagHandler<T> *fh = new (GetGlobalLowLevelAllocator()) FlagHandler<T>(var); parser->RegisterHandler(name, fh, desc); } diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.cpp b/libsanitizer/sanitizer_common/sanitizer_flags.cpp index d52e96a7c38..849a122386a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_flags.cpp @@ -108,11 +108,11 @@ class FlagHandlerInclude final : public FlagHandlerBase { }; void RegisterIncludeFlags(FlagParser *parser, CommonFlags *cf) { - FlagHandlerInclude *fh_include = new (FlagParser::Alloc) + FlagHandlerInclude *fh_include = new (GetGlobalLowLevelAllocator()) FlagHandlerInclude(parser, /*ignore_missing*/ false); parser->RegisterHandler("include", fh_include, "read more options from the given file"); - FlagHandlerInclude *fh_include_if_exists = new (FlagParser::Alloc) + FlagHandlerInclude *fh_include_if_exists = new (GetGlobalLowLevelAllocator()) FlagHandlerInclude(parser, /*ignore_missing*/ true); parser->RegisterHandler( "include_if_exists", fh_include_if_exists, diff --git a/libsanitizer/sanitizer_common/sanitizer_flags.inc b/libsanitizer/sanitizer_common/sanitizer_flags.inc index 6148ae56067..949bdbd148b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flags.inc +++ b/libsanitizer/sanitizer_common/sanitizer_flags.inc @@ -269,3 +269,9 @@ COMMON_FLAG(bool, detect_write_exec, false, COMMON_FLAG(bool, test_only_emulate_no_memorymap, false, "TEST ONLY fail to read memory mappings to emulate sanitized " "\"init\"") +// With static linking, dladdr((void*)pthread_join) or similar will return the +// path to the main program. This flag will replace dlopen(<main program,...> +// with dlopen(NULL,...), which is the correct way to get a handle to the main +// program. +COMMON_FLAG(bool, test_only_replace_dlopen_main_program, false, + "TEST ONLY replace dlopen(<main program>,...) with dlopen(NULL)") diff --git a/libsanitizer/sanitizer_common/sanitizer_flat_map.h b/libsanitizer/sanitizer_common/sanitizer_flat_map.h index 05fb554d20c..8bb8304910c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_flat_map.h +++ b/libsanitizer/sanitizer_common/sanitizer_flat_map.h @@ -21,12 +21,6 @@ namespace __sanitizer { -// Call these callbacks on mmap/munmap. -struct NoOpMapUnmapCallback { - void OnMap(uptr p, uptr size) const {} - void OnUnmap(uptr p, uptr size) const {} -}; - // Maps integers in rage [0, kSize) to values. template <typename T, u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView> @@ -62,8 +56,7 @@ class FlatMap { // Each value is initially zero and can be set to something else only once. // Setting and getting values from multiple threads is safe w/o extra locking. template <typename T, u64 kSize1, u64 kSize2, - typename AddressSpaceViewTy = LocalAddressSpaceView, - class MapUnmapCallback = NoOpMapUnmapCallback> + typename AddressSpaceViewTy = LocalAddressSpaceView> class TwoLevelMap { static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance."); @@ -79,7 +72,6 @@ class TwoLevelMap { T *p = Get(i); if (!p) continue; - MapUnmapCallback().OnUnmap(reinterpret_cast<uptr>(p), MmapSize()); UnmapOrDie(p, kSize2); } Init(); @@ -149,7 +141,6 @@ class TwoLevelMap { T *res = Get(idx); if (!res) { res = reinterpret_cast<T *>(MmapOrDie(MmapSize(), "TwoLevelMap")); - MapUnmapCallback().OnMap(reinterpret_cast<uptr>(res), kSize2); atomic_store(&map1_[idx], reinterpret_cast<uptr>(res), memory_order_release); } @@ -164,10 +155,8 @@ template <u64 kSize, typename AddressSpaceViewTy = LocalAddressSpaceView> using FlatByteMap = FlatMap<u8, kSize, AddressSpaceViewTy>; template <u64 kSize1, u64 kSize2, - typename AddressSpaceViewTy = LocalAddressSpaceView, - class MapUnmapCallback = NoOpMapUnmapCallback> -using TwoLevelByteMap = - TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy, MapUnmapCallback>; + typename AddressSpaceViewTy = LocalAddressSpaceView> +using TwoLevelByteMap = TwoLevelMap<u8, kSize1, kSize2, AddressSpaceViewTy>; } // namespace __sanitizer #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp index a92e84cb8ec..0245164403c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp @@ -226,13 +226,14 @@ static uptr DoMmapFixedOrDie(zx_handle_t vmar, uptr fixed_addr, uptr map_size, uptr ReservedAddressRange::Map(uptr fixed_addr, uptr map_size, const char *name) { - return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, - false); + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name ? name : name_, false); } uptr ReservedAddressRange::MapOrDie(uptr fixed_addr, uptr map_size, const char *name) { - return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, name_, true); + return DoMmapFixedOrDie(os_handle_, fixed_addr, map_size, base_, + name ? name : name_, true); } void UnmapOrDieVmar(void *addr, uptr size, zx_handle_t target_vmar) { @@ -285,6 +286,12 @@ bool MprotectReadOnly(uptr addr, uptr size) { ZX_OK; } +bool MprotectReadWrite(uptr addr, uptr size) { + return _zx_vmar_protect(_zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, addr, + size) == ZX_OK; +} + void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, const char *mem_type) { CHECK_GE(size, GetPageSize()); diff --git a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h index 98186c429e9..3809669dd48 100644 --- a/libsanitizer/sanitizer_common/sanitizer_internal_defs.h +++ b/libsanitizer/sanitizer_common/sanitizer_internal_defs.h @@ -13,6 +13,12 @@ #define SANITIZER_DEFS_H #include "sanitizer_platform.h" +#include "sanitizer_redefine_builtins.h" + +// GCC does not understand __has_feature. +#if !defined(__has_feature) +#define __has_feature(x) 0 +#endif #ifndef SANITIZER_DEBUG # define SANITIZER_DEBUG 0 @@ -217,7 +223,7 @@ typedef u64 tid_t; # define WARN_UNUSED_RESULT #else // _MSC_VER # define ALWAYS_INLINE inline __attribute__((always_inline)) -# define ALIAS(x) __attribute__((alias(x))) +# define ALIAS(x) __attribute__((alias(SANITIZER_STRINGIFY(x)))) // Please only use the ALIGNED macro before the type. // Using ALIGNED after the variable declaration is not portable! # define ALIGNED(x) __attribute__((aligned(x))) @@ -258,6 +264,12 @@ typedef u64 tid_t; # define FALLTHROUGH #endif +#if __has_attribute(uninitialized) +# define UNINITIALIZED __attribute__((uninitialized)) +#else +# define UNINITIALIZED +#endif + // Unaligned versions of basic types. typedef ALIGNED(1) u16 uu16; typedef ALIGNED(1) u32 uu32; diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.cpp b/libsanitizer/sanitizer_common/sanitizer_libc.cpp index d3076f0da48..9318066afed 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libc.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_libc.cpp @@ -10,6 +10,9 @@ // run-time libraries. See sanitizer_libc.h for details. //===----------------------------------------------------------------------===// +// Do not redefine builtins; this file is defining the builtin replacements. +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_libc.h" @@ -46,7 +49,10 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) { return 0; } -void *internal_memcpy(void *dest, const void *src, uptr n) { +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest, + const void *src, + uptr n) { char *d = (char*)dest; const char *s = (const char *)src; for (uptr i = 0; i < n; ++i) @@ -54,7 +60,8 @@ void *internal_memcpy(void *dest, const void *src, uptr n) { return dest; } -void *internal_memmove(void *dest, const void *src, uptr n) { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove( + void *dest, const void *src, uptr n) { char *d = (char*)dest; const char *s = (const char *)src; sptr i, signed_n = (sptr)n; @@ -72,7 +79,8 @@ void *internal_memmove(void *dest, const void *src, uptr n) { return dest; } -void *internal_memset(void* s, int c, uptr n) { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c, + uptr n) { // Optimize for the most performance-critical case: if ((reinterpret_cast<uptr>(s) % 16) == 0 && (n % 16) == 0) { u64 *p = reinterpret_cast<u64*>(s); @@ -95,6 +103,7 @@ void *internal_memset(void* s, int c, uptr n) { } return s; } +} // extern "C" uptr internal_strcspn(const char *s, const char *reject) { uptr i; @@ -190,6 +199,14 @@ char *internal_strncat(char *dst, const char *src, uptr n) { return dst; } +wchar_t *internal_wcscpy(wchar_t *dst, const wchar_t *src) { + wchar_t *dst_it = dst; + do { + *dst_it++ = *src++; + } while (*src); + return dst; +} + uptr internal_strlcpy(char *dst, const char *src, uptr maxlen) { const uptr srclen = internal_strlen(src); if (srclen < maxlen) { @@ -209,6 +226,14 @@ char *internal_strncpy(char *dst, const char *src, uptr n) { return dst; } +wchar_t *internal_wcsncpy(wchar_t *dst, const wchar_t *src, uptr n) { + uptr i; + for (i = 0; i < n && src[i]; ++i) + dst[i] = src[i]; + internal_memset(dst + i, 0, (n - i) * sizeof(wchar_t)); + return dst; +} + uptr internal_strnlen(const char *s, uptr maxlen) { uptr i = 0; while (i < maxlen && s[i]) i++; diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.h b/libsanitizer/sanitizer_common/sanitizer_libc.h index 39a212665d0..1906569e2a5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libc.h +++ b/libsanitizer/sanitizer_common/sanitizer_libc.h @@ -24,15 +24,33 @@ namespace __sanitizer { // internal_X() is a custom implementation of X() for use in RTL. +extern "C" { +// These are used as builtin replacements; see sanitizer_redefine_builtins.h. +// In normal runtime code, use the __sanitizer::internal_X() aliases instead. +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest, + const void *src, + uptr n); +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove( + void *dest, const void *src, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c, + uptr n); +} // extern "C" + // String functions s64 internal_atoll(const char *nptr); void *internal_memchr(const void *s, int c, uptr n); void *internal_memrchr(const void *s, int c, uptr n); int internal_memcmp(const void* s1, const void* s2, uptr n); -void *internal_memcpy(void *dest, const void *src, uptr n); -void *internal_memmove(void *dest, const void *src, uptr n); +ALWAYS_INLINE void *internal_memcpy(void *dest, const void *src, uptr n) { + return __sanitizer_internal_memcpy(dest, src, n); +} +ALWAYS_INLINE void *internal_memmove(void *dest, const void *src, uptr n) { + return __sanitizer_internal_memmove(dest, src, n); +} // Should not be used in performance-critical places. -void *internal_memset(void *s, int c, uptr n); +ALWAYS_INLINE void *internal_memset(void *s, int c, uptr n) { + return __sanitizer_internal_memset(s, c, n); +} char* internal_strchr(const char *s, int c); char *internal_strchrnul(const char *s, int c); int internal_strcmp(const char *s1, const char *s2); @@ -53,7 +71,8 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...) FORMAT(3, 4); uptr internal_wcslen(const wchar_t *s); uptr internal_wcsnlen(const wchar_t *s, uptr maxlen); - +wchar_t *internal_wcscpy(wchar_t *dst, const wchar_t *src); +wchar_t *internal_wcsncpy(wchar_t *dst, const wchar_t *src, uptr maxlen); // Return true if all bytes in [mem, mem+size) are zero. // Optimized for the case when the result is true. bool mem_is_zero(const char *mem, uptr size); diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_linux.cpp index 24c6acaa9e5..d2b3b63f3a7 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux.cpp @@ -156,11 +156,11 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; namespace __sanitizer { -void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *old) { - CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, old)); +void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset)); } -ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { +void BlockSignals(__sanitizer_sigset_t *oldset) { __sanitizer_sigset_t set; internal_sigfillset(&set); # if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -175,7 +175,11 @@ ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { // hang. internal_sigdelset(&set, 31); # endif - SetSigProcMask(&set, &saved_); + SetSigProcMask(&set, oldset); +} + +ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { + BlockSignals(&saved_); if (copy) internal_memcpy(copy, &saved_, sizeof(saved_)); } diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.h b/libsanitizer/sanitizer_common/sanitizer_linux.h index c84c04a8775..7454369fa41 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.h +++ b/libsanitizer/sanitizer_common/sanitizer_linux.h @@ -51,6 +51,7 @@ uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); +void BlockSignals(__sanitizer_sigset_t *oldset = nullptr); struct ScopedBlockSignals { explicit ScopedBlockSignals(__sanitizer_sigset_t *copy); ~ScopedBlockSignals(); diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp index 2720a3cab2c..fcfaa0c36c2 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -148,7 +148,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, pthread_attr_t attr; pthread_attr_init(&attr); CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); - my_pthread_attr_getstack(&attr, &stackaddr, &stacksize); + internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); #endif // SANITIZER_SOLARIS @@ -698,11 +698,8 @@ static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { return AddModuleSegments(module_name.data(), info, data->modules); } - if (info->dlpi_name) { - InternalScopedString module_name; - module_name.append("%s", info->dlpi_name); - return AddModuleSegments(module_name.data(), info, data->modules); - } + if (info->dlpi_name) + return AddModuleSegments(info->dlpi_name, info, data->modules); return 0; } @@ -838,13 +835,9 @@ u32 GetNumberOfCPUs() { #elif SANITIZER_SOLARIS return sysconf(_SC_NPROCESSORS_ONLN); #else -#if defined(CPU_COUNT) cpu_set_t CPUs; CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); return CPU_COUNT(&CPUs); -#else - return 1; -#endif #endif } diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_mac.cpp index e1f83e4002a..24e3d111252 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_mac.cpp @@ -38,7 +38,7 @@ extern char **environ; # endif -# if defined(__has_include) && __has_include(<os/trace.h>) && defined(__BLOCKS__) +# if defined(__has_include) && __has_include(<os/trace.h>) # define SANITIZER_OS_TRACE 1 # include <os/trace.h> # else @@ -71,15 +71,7 @@ extern char ***_NSGetArgv(void); # include <mach/mach_time.h> # include <mach/vm_statistics.h> # include <malloc/malloc.h> -# if defined(__has_builtin) && __has_builtin(__builtin_os_log_format) -# include <os/log.h> -# else - /* Without support for __builtin_os_log_format, fall back to the older - method. */ -# define OS_LOG_DEFAULT 0 -# define os_log_error(A,B,C) \ - asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", (C)); -# endif +# include <os/log.h> # include <pthread.h> # include <pthread/introspection.h> # include <sched.h> diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h index 1cf2e298cc9..f0a97d098ee 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.h +++ b/libsanitizer/sanitizer_common/sanitizer_mac.h @@ -14,26 +14,6 @@ #include "sanitizer_common.h" #include "sanitizer_platform.h" - -/* TARGET_OS_OSX is not present in SDKs before Darwin16 (macOS 10.12) use - TARGET_OS_MAC (we have no support for iOS in any form for these versions, - so there's no ambiguity). */ -#if !defined(TARGET_OS_OSX) && TARGET_OS_MAC -# define TARGET_OS_OSX 1 -#endif - -/* Other TARGET_OS_xxx are not present on earlier versions, define them to - 0 (we have no support for them; they are not valid targets anyway). */ -#ifndef TARGET_OS_IOS -#define TARGET_OS_IOS 0 -#endif -#ifndef TARGET_OS_TV -#define TARGET_OS_TV 0 -#endif -#ifndef TARGET_OS_WATCH -#define TARGET_OS_WATCH 0 -#endif - #if SANITIZER_APPLE #include "sanitizer_posix.h" diff --git a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc index fe76b3f8aa0..6343eb284af 100644 --- a/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc +++ b/libsanitizer/sanitizer_common/sanitizer_malloc_mac.inc @@ -123,7 +123,7 @@ INTERCEPTOR(void, malloc_set_zone_name, malloc_zone_t *zone, const char *name) { COMMON_MALLOC_ENTER(); InternalScopedString new_name; if (name && zone->introspect == sanitizer_zone.introspect) { - new_name.append(COMMON_MALLOC_ZONE_NAME "-%s", name); + new_name.AppendF(COMMON_MALLOC_ZONE_NAME "-%s", name); name = new_name.data(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h index 764996e5735..3e1b078a021 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform.h @@ -284,7 +284,8 @@ // For such platforms build this code with -DSANITIZER_CAN_USE_ALLOCATOR64=0 or // change the definition of SANITIZER_CAN_USE_ALLOCATOR64 here. #ifndef SANITIZER_CAN_USE_ALLOCATOR64 -# if SANITIZER_RISCV64 || SANITIZER_IOS +# if (SANITIZER_RISCV64 && !SANITIZER_FUCHSIA) || SANITIZER_IOS || \ + SANITIZER_DRIVERKIT # define SANITIZER_CAN_USE_ALLOCATOR64 0 # elif defined(__mips64) || defined(__hexagon__) # define SANITIZER_CAN_USE_ALLOCATOR64 0 @@ -303,7 +304,15 @@ # define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 40) # endif #elif SANITIZER_RISCV64 -# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 38) +// FIXME: Rather than hardcoding the VMA here, we should rely on +// GetMaxUserVirtualAddress(). This will require some refactoring though since +// many places either hardcode some value or SANITIZER_MMAP_RANGE_SIZE is +// assumed to be some constant integer. +# if SANITIZER_FUCHSIA +# define SANITIZER_MMAP_RANGE_SIZE (1ULL << 38) +# else +# define SANITIZER_MMAP_RANGE_SIZE FIRST_32_SECOND_64(1ULL << 32, 1ULL << 47) +# endif #elif defined(__aarch64__) # if SANITIZER_APPLE # if SANITIZER_OSX || SANITIZER_IOSSIM diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h index c82ab5c2105..8c7c00de6d1 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h @@ -347,7 +347,8 @@ #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC -#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP \ + (SI_LINUX_NOT_ANDROID || SI_FREEBSD) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \ (SI_POSIX && !SI_NETBSD) @@ -367,6 +368,8 @@ (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \ (SI_LINUX_NOT_ANDROID && !SI_NETBSD) +#define SANITIZER_INTERCEPT_TRYJOIN SI_GLIBC +#define SANITIZER_INTERCEPT_TIMEDJOIN SI_GLIBC #define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD #define SANITIZER_INTERCEPT_TMPNAM SI_POSIX #define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC || SI_SOLARIS) @@ -492,6 +495,7 @@ #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WCSLEN 1 #define SANITIZER_INTERCEPT_WCSCAT SI_POSIX #define SANITIZER_INTERCEPT_WCSDUP SI_POSIX #define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA) @@ -571,11 +575,12 @@ #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD) #define SANITIZER_INTERCEPT_GETRANDOM \ - ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD) + ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD #define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD #define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD -#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD +#define SANITIZER_INTERCEPT_GETENTROPY \ + ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD || SI_SOLARIS) #define SANITIZER_INTERCEPT_QSORT \ (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID) #define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC @@ -591,6 +596,7 @@ #define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD #define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD #define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC +#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD // This macro gives a way for downstream users to override the above // interceptor macros irrespective of the platform they are on. They have diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp index 37e72cd5d45..38f968d533b 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.cpp @@ -17,6 +17,7 @@ #include <sys/capsicum.h> #include <sys/consio.h> +#include <sys/cpuset.h> #include <sys/filio.h> #include <sys/ipc.h> #include <sys/kbio.h> @@ -103,6 +104,7 @@ void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) { return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr; } +unsigned struct_cpuset_sz = sizeof(cpuset_t); unsigned struct_cap_rights_sz = sizeof(cap_rights_t); unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); @@ -173,6 +175,12 @@ uptr __sanitizer_in_addr_sz(int af) { return 0; } +// For FreeBSD the actual size of a directory entry is not always in d_reclen. +// Use the appropriate macro to get the correct size for all cases (e.g. NFS). +u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp) { + return _GENERIC_DIRSIZ(dp); +} + unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); int glob_nomatch = GLOB_NOMATCH; int glob_altdirfunc = GLOB_ALTDIRFUNC; @@ -558,4 +566,5 @@ COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); CHECK_TYPE_SIZE(sem_t); COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t)); +COMPILER_CHECK(sizeof(__sanitizer_cpuset_t) >= sizeof(cpuset_t)); #endif // SANITIZER_FREEBSD diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h index daef1177a2d..b119f059007 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -249,9 +249,15 @@ struct __sanitizer_dirent { unsigned int d_fileno; # endif unsigned short d_reclen; - // more fields that we don't care about + u8 d_type; + u8 d_pad0; + u16 d_namlen; + u16 d_pad1; + char d_name[256]; }; +u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp); + // 'clock_t' is 32 bits wide on x64 FreeBSD typedef int __sanitizer_clock_t; typedef int __sanitizer_clockid_t; @@ -709,6 +715,17 @@ extern unsigned struct_cap_rights_sz; extern unsigned struct_fstab_sz; extern unsigned struct_StringList_sz; + +struct __sanitizer_cpuset { +#if __FreeBSD_version >= 1400090 + long __bits[(1024 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)]; +#else + long __bits[(256 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)]; +#endif +}; + +typedef struct __sanitizer_cpuset __sanitizer_cpuset_t; +extern unsigned struct_cpuset_sz; } // namespace __sanitizer # define CHECK_TYPE_SIZE(TYPE) \ diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp index c278c8797f7..bf0f355847c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp @@ -26,10 +26,7 @@ // With old kernels (and even new kernels on powerpc) asm/stat.h uses types that // are not defined anywhere in userspace headers. Fake them. This seems to work -// fine with newer headers, too. Beware that with <sys/stat.h>, struct stat -// takes the form of struct stat64 on 32-bit platforms if _FILE_OFFSET_BITS=64. -// Also, for some platforms (e.g. mips) there are additional members in the -// <sys/stat.h> struct stat:s. +// fine with newer headers, too. #include <linux/posix_types.h> # if defined(__x86_64__) || defined(__mips__) || defined(__hexagon__) # include <sys/stat.h> diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.cpp b/libsanitizer/sanitizer_common/sanitizer_posix.cpp index f6b0bbd3a5d..8d2c5b2cefb 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_posix.cpp @@ -154,6 +154,10 @@ bool MprotectReadOnly(uptr addr, uptr size) { return 0 == internal_mprotect((void *)addr, size, PROT_READ); } +bool MprotectReadWrite(uptr addr, uptr size) { + return 0 == internal_mprotect((void *)addr, size, PROT_READ | PROT_WRITE); +} + #if !SANITIZER_APPLE void MprotectMallocZones(void *addr, int prot) {} #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_posix.h b/libsanitizer/sanitizer_common/sanitizer_posix.h index f91e26e74b8..c5811dffea9 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix.h +++ b/libsanitizer/sanitizer_common/sanitizer_posix.h @@ -90,7 +90,7 @@ int real_pthread_join(void *th, void **ret); } \ } // namespace __sanitizer -int my_pthread_attr_getstack(void *attr, void **addr, uptr *size); +int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size); // A routine named real_sigaction() must be implemented by each sanitizer in // order for internal_sigaction() to bypass interceptors. @@ -120,6 +120,9 @@ int GetNamedMappingFd(const char *name, uptr size, int *flags); // alive at least as long as the mapping exists. void DecorateMapping(uptr addr, uptr size, const char *name); +# if !SANITIZER_FREEBSD +# define __sanitizer_dirsiz(dp) ((dp)->d_reclen) +# endif } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp index 46e41c66973..e88e654eec5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -383,7 +383,7 @@ SANITIZER_WEAK_ATTRIBUTE int real_pthread_attr_getstack(void *attr, void **addr, size_t *size); } // extern "C" -int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) { +int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size) { #if !SANITIZER_GO && !SANITIZER_APPLE if (&real_pthread_attr_getstack) return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, @@ -397,7 +397,7 @@ void AdjustStackSize(void *attr_) { pthread_attr_t *attr = (pthread_attr_t *)attr_; uptr stackaddr = 0; uptr stacksize = 0; - my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize); + internal_pthread_attr_getstack(attr, (void **)&stackaddr, &stacksize); // GLibC will return (0 - stacksize) as the stack address in the case when // stacksize is set, but stackaddr is not. bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0); diff --git a/libsanitizer/sanitizer_common/sanitizer_printf.cpp b/libsanitizer/sanitizer_common/sanitizer_printf.cpp index 3a9e366d2df..62c1cf4abe4 100644 --- a/libsanitizer/sanitizer_common/sanitizer_printf.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_printf.cpp @@ -337,7 +337,14 @@ int internal_snprintf(char *buffer, uptr length, const char *format, ...) { return needed_length; } -void InternalScopedString::append(const char *format, ...) { +void InternalScopedString::Append(const char *str) { + uptr prev_len = length(); + uptr str_len = internal_strlen(str); + buffer_.resize(prev_len + str_len + 1); + internal_memcpy(buffer_.data() + prev_len, str, str_len + 1); +} + +void InternalScopedString::AppendF(const char *format, ...) { uptr prev_len = length(); while (true) { diff --git a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp index f2f38467121..b44e016a0e5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_procmaps_mac.cpp @@ -146,13 +146,8 @@ static bool IsDyldHdr(const mach_header *hdr) { // until we hit a Mach header matching dyld instead. These recurse // calls are expensive, but the first memory map generation occurs // early in the process, when dyld is one of the only images loaded, -// so it will be hit after only a few iterations. These assumptions don't -// hold on macOS 13+ anymore (dyld itself has moved into the shared cache). - -// FIXME: Unfortunately, the upstream revised version to deal with macOS 13+ -// is incompatible with GCC and also uses APIs not available on earlier -// systems which we support; backed out for now. - +// so it will be hit after only a few iterations. These assumptions don't hold +// on macOS 13+ anymore (dyld itself has moved into the shared cache). static mach_header *GetDyldImageHeaderViaVMRegion() { vm_address_t address = 0; @@ -176,17 +171,64 @@ static mach_header *GetDyldImageHeaderViaVMRegion() { } } +extern "C" { +struct dyld_shared_cache_dylib_text_info { + uint64_t version; // current version 2 + // following fields all exist in version 1 + uint64_t loadAddressUnslid; + uint64_t textSegmentSize; + uuid_t dylibUuid; + const char *path; // pointer invalid at end of iterations + // following fields all exist in version 2 + uint64_t textSegmentOffset; // offset from start of cache +}; +typedef struct dyld_shared_cache_dylib_text_info + dyld_shared_cache_dylib_text_info; + +extern bool _dyld_get_shared_cache_uuid(uuid_t uuid); +extern const void *_dyld_get_shared_cache_range(size_t *length); +extern int dyld_shared_cache_iterate_text( + const uuid_t cacheUuid, + void (^callback)(const dyld_shared_cache_dylib_text_info *info)); +} // extern "C" + +static mach_header *GetDyldImageHeaderViaSharedCache() { + uuid_t uuid; + bool hasCache = _dyld_get_shared_cache_uuid(uuid); + if (!hasCache) + return nullptr; + + size_t cacheLength; + __block uptr cacheStart = (uptr)_dyld_get_shared_cache_range(&cacheLength); + CHECK(cacheStart && cacheLength); + + __block mach_header *dyldHdr = nullptr; + int res = dyld_shared_cache_iterate_text( + uuid, ^(const dyld_shared_cache_dylib_text_info *info) { + CHECK_GE(info->version, 2); + mach_header *hdr = + (mach_header *)(cacheStart + info->textSegmentOffset); + if (IsDyldHdr(hdr)) + dyldHdr = hdr; + }); + CHECK_EQ(res, 0); + + return dyldHdr; +} + const mach_header *get_dyld_hdr() { if (!dyld_hdr) { // On macOS 13+, dyld itself has moved into the shared cache. Looking it up // via vm_region_recurse_64() causes spins/hangs/crashes. - // FIXME: find a way to do this compatible with GCC. if (GetMacosAlignedVersion() >= MacosVersion(13, 0)) { + dyld_hdr = GetDyldImageHeaderViaSharedCache(); + if (!dyld_hdr) { VReport(1, - "looking up the dyld image header in the shared cache on " - "macOS 13+ is not yet supported. Falling back to " + "Failed to lookup the dyld image header in the shared cache on " + "macOS 13+ (or no shared cache in use). Falling back to " "lookup via vm_region_recurse_64().\n"); dyld_hdr = GetDyldImageHeaderViaVMRegion(); + } } else { dyld_hdr = GetDyldImageHeaderViaVMRegion(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_quarantine.h b/libsanitizer/sanitizer_common/sanitizer_quarantine.h index 4aa60548516..460d96ea681 100644 --- a/libsanitizer/sanitizer_common/sanitizer_quarantine.h +++ b/libsanitizer/sanitizer_common/sanitizer_quarantine.h @@ -68,10 +68,6 @@ struct QuarantineBatch { COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13)); // 8Kb. -// The callback interface is: -// void Callback::Recycle(Node *ptr); -// void *cb.Allocate(uptr size); -// void cb.Deallocate(void *ptr); template<typename Callback, typename Node> class Quarantine { public: @@ -94,21 +90,20 @@ class Quarantine { recycle_mutex_.Init(); } - uptr GetSize() const { return atomic_load_relaxed(&max_size_); } - uptr GetCacheSize() const { - return atomic_load_relaxed(&max_cache_size_); - } + uptr GetMaxSize() const { return atomic_load_relaxed(&max_size_); } + uptr GetMaxCacheSize() const { return atomic_load_relaxed(&max_cache_size_); } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { - uptr cache_size = GetCacheSize(); - if (cache_size) { + uptr max_cache_size = GetMaxCacheSize(); + if (max_cache_size && size <= GetMaxSize()) { + cb.PreQuarantine(ptr); c->Enqueue(cb, ptr, size); } else { - // GetCacheSize() == 0 only when GetSize() == 0 (see Init). - cb.Recycle(ptr); + // GetMaxCacheSize() == 0 only when GetMaxSize() == 0 (see Init). + cb.RecyclePassThrough(ptr); } // Check cache size anyway to accommodate for runtime cache_size change. - if (c->Size() > cache_size) + if (c->Size() > max_cache_size) Drain(c, cb); } @@ -117,7 +112,7 @@ class Quarantine { SpinMutexLock l(&cache_mutex_); cache_.Transfer(c); } - if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) + if (cache_.Size() > GetMaxSize() && recycle_mutex_.TryLock()) Recycle(atomic_load_relaxed(&min_size_), cb); } @@ -133,7 +128,7 @@ class Quarantine { void PrintStats() const { // It assumes that the world is stopped, just as the allocator's PrintStats. Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n", - GetSize() >> 20, GetCacheSize() >> 10); + GetMaxSize() >> 20, GetMaxCacheSize() >> 10); cache_.PrintStats(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_range.cpp b/libsanitizer/sanitizer_common/sanitizer_range.cpp new file mode 100644 index 00000000000..68d79f18ac8 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_range.cpp @@ -0,0 +1,62 @@ +//===-- sanitizer_range.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_range.h" + +#include "sanitizer_common/sanitizer_array_ref.h" + +namespace __sanitizer { + +void Intersect(ArrayRef<Range> a, ArrayRef<Range> b, + InternalMmapVectorNoCtor<Range> &output) { + output.clear(); + + struct Event { + uptr val; + s8 diff1; + s8 diff2; + }; + + InternalMmapVector<Event> events; + for (const Range &r : a) { + CHECK_LE(r.begin, r.end); + events.push_back({r.begin, 1, 0}); + events.push_back({r.end, -1, 0}); + } + + for (const Range &r : b) { + CHECK_LE(r.begin, r.end); + events.push_back({r.begin, 0, 1}); + events.push_back({r.end, 0, -1}); + } + + Sort(events.data(), events.size(), + [](const Event &lh, const Event &rh) { return lh.val < rh.val; }); + + uptr start = 0; + sptr state1 = 0; + sptr state2 = 0; + for (const auto &e : events) { + if (e.val != start) { + DCHECK_GE(state1, 0); + DCHECK_GE(state2, 0); + if (state1 && state2) { + if (!output.empty() && start == output.back().end) + output.back().end = e.val; + else + output.push_back({start, e.val}); + } + start = e.val; + } + + state1 += e.diff1; + state2 += e.diff2; + } +} + +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_range.h b/libsanitizer/sanitizer_common/sanitizer_range.h new file mode 100644 index 00000000000..7c593e171ba --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_range.h @@ -0,0 +1,40 @@ +//===-- sanitizer_range.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contais Range and related utilities. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_RANGE_H +#define SANITIZER_RANGE_H + +#include "sanitizer_common.h" +#include "sanitizer_common/sanitizer_array_ref.h" + +namespace __sanitizer { + +struct Range { + uptr begin; + uptr end; +}; + +inline bool operator==(const Range &lhs, const Range &rhs) { + return lhs.begin == rhs.begin && lhs.end == rhs.end; +} + +inline bool operator!=(const Range &lhs, const Range &rhs) { + return !(lhs == rhs); +} + +// Calculates intersection of two sets of regions in O(N log N) time. +void Intersect(ArrayRef<Range> a, ArrayRef<Range> b, + InternalMmapVectorNoCtor<Range> &output); + +} // namespace __sanitizer + +#endif // SANITIZER_RANGE_H diff --git a/libsanitizer/sanitizer_common/sanitizer_redefine_builtins.h b/libsanitizer/sanitizer_common/sanitizer_redefine_builtins.h new file mode 100644 index 00000000000..d24b179ef32 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_redefine_builtins.h @@ -0,0 +1,56 @@ +//===-- sanitizer_redefine_builtins.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Redefine builtin functions to use internal versions. This is needed where +// compiler optimizations end up producing unwanted libcalls! +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS +# ifndef SANITIZER_REDEFINE_BUILTINS_H +# define SANITIZER_REDEFINE_BUILTINS_H + +// The asm hack only works with GCC and Clang. +# if !defined(_WIN32) + +asm("memcpy = __sanitizer_internal_memcpy"); +asm("memmove = __sanitizer_internal_memmove"); +asm("memset = __sanitizer_internal_memset"); + +# if defined(__cplusplus) && \ + !defined(SANITIZER_COMMON_REDEFINE_BUILTINS_IN_STD) + +// The builtins should not be redefined in source files that make use of C++ +// standard libraries, in particular where C++STL headers with inline functions +// are used. The redefinition in such cases would lead to ODR violations. +// +// Try to break the build in common cases where builtins shouldn't be redefined. +namespace std { +class Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file { + Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file( + const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete; + Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file& operator=( + const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete; +}; +using array = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using atomic = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using function = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using shared_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using string = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unique_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unordered_map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +} // namespace std + +# endif // __cpluplus +# endif // !_WIN32 + +# endif // SANITIZER_REDEFINE_BUILTINS_H +#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS diff --git a/libsanitizer/sanitizer_common/sanitizer_ring_buffer.h b/libsanitizer/sanitizer_common/sanitizer_ring_buffer.h index f22e40cac28..6222a958b11 100644 --- a/libsanitizer/sanitizer_common/sanitizer_ring_buffer.h +++ b/libsanitizer/sanitizer_common/sanitizer_ring_buffer.h @@ -47,7 +47,9 @@ class RingBuffer { void push(T t) { *next_ = t; next_--; - // The condition below works only if sizeof(T) is divisible by sizeof(T*). + static_assert((sizeof(T) % sizeof(T *)) == 0, + "The condition below works only if sizeof(T) is divisible by " + "sizeof(T*)."); if (next_ <= reinterpret_cast<T*>(&next_)) next_ = last_; } diff --git a/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc index 475e577d998..94e4e2954a3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc +++ b/libsanitizer/sanitizer_common/sanitizer_signal_interceptors.inc @@ -43,6 +43,7 @@ using namespace __sanitizer; #if SANITIZER_INTERCEPT_BSD_SIGNAL INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler); } @@ -53,6 +54,7 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { #if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION INTERCEPTOR(uptr, signal, int signum, uptr handler) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return (uptr) nullptr; SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler); @@ -61,6 +63,7 @@ INTERCEPTOR(uptr, signal, int signum, uptr handler) { INTERCEPTOR(int, sigaction_symname, int signum, const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) { if (!oldact) return 0; act = nullptr; diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp index 661495e2340..d24fae98213 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp @@ -87,8 +87,8 @@ static inline uhwptr *GetCanonicFrame(uptr bp, // Nope, this does not look right either. This means the frame after next does // not have a valid frame pointer, but we can still extract the caller PC. // Unfortunately, there is no way to decide between GCC and LLVM frame - // layouts. Assume GCC. - return bp_prev - 1; + // layouts. Assume LLVM. + return bp_prev; #else return (uhwptr*)bp; #endif @@ -111,21 +111,14 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, IsAligned((uptr)frame, sizeof(*frame)) && size < max_depth) { #ifdef __powerpc__ - // PowerPC ABIs specify that the return address is saved on the - // *caller's* stack frame. Thus we must dereference the back chain - // to find the caller frame before extracting it. + // PowerPC ABIs specify that the return address is saved at offset + // 16 of the *caller's* stack frame. Thus we must dereference the + // back chain to find the caller frame before extracting it. uhwptr *caller_frame = (uhwptr*)frame[0]; if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) || !IsAligned((uptr)caller_frame, sizeof(uhwptr))) break; - // For most ABIs the offset where the return address is saved is two - // register sizes. The exception is the SVR4 ABI, which uses an - // offset of only one register size. -#ifdef _CALL_SYSV - uhwptr pc1 = caller_frame[1]; -#else uhwptr pc1 = caller_frame[2]; -#endif #elif defined(__s390__) uhwptr pc1 = frame[14]; #elif defined(__loongarch__) || defined(__riscv) diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp index 47983ee7ec7..9a4c80fcfdd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp @@ -29,7 +29,8 @@ class StackTraceTextPrinter { frame_delimiter_(frame_delimiter), output_(output), dedup_token_(dedup_token), - symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {} + symbolize_(StackTracePrinter::GetOrInit()->RenderNeedsSymbolization( + stack_trace_fmt)) {} bool ProcessAddressFrames(uptr pc) { SymbolizedStack *frames = symbolize_ @@ -40,13 +41,13 @@ class StackTraceTextPrinter { for (SymbolizedStack *cur = frames; cur; cur = cur->next) { uptr prev_len = output_->length(); - RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address, - symbolize_ ? &cur->info : nullptr, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderFrame( + output_, stack_trace_fmt_, frame_num_++, cur->info.address, + symbolize_ ? &cur->info : nullptr, common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); if (prev_len != output_->length()) - output_->append("%c", frame_delimiter_); + output_->AppendF("%c", frame_delimiter_); ExtendDedupToken(cur); } @@ -62,9 +63,9 @@ class StackTraceTextPrinter { if (dedup_frames_-- > 0) { if (dedup_token_->length()) - dedup_token_->append("--"); - if (stack->info.function != nullptr) - dedup_token_->append("%s", stack->info.function); + dedup_token_->AppendF("--"); + if (stack->info.function) + dedup_token_->Append(stack->info.function); } } @@ -98,7 +99,7 @@ void StackTrace::PrintTo(InternalScopedString *output) const { output, &dedup_token); if (trace == nullptr || size == 0) { - output->append(" <empty stack>\n\n"); + output->AppendF(" <empty stack>\n\n"); return; } @@ -110,11 +111,11 @@ void StackTrace::PrintTo(InternalScopedString *output) const { } // Always add a trailing empty line after stack trace. - output->append("\n"); + output->AppendF("\n"); // Append deduplication token, if non-empty. if (dedup_token.length()) - output->append("DEDUP_TOKEN: %s\n", dedup_token.data()); + output->AppendF("DEDUP_TOKEN: %s\n", dedup_token.data()); } uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const { @@ -197,7 +198,7 @@ void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf, StackTraceTextPrinter printer(fmt, '\0', &output, nullptr); if (!printer.ProcessAddressFrames(pc)) { output.clear(); - output.append("<can't symbolize>"); + output.AppendF("<can't symbolize>"); } CopyStringToBuffer(output, out_buf, out_buf_size); } @@ -210,7 +211,8 @@ void __sanitizer_symbolize_global(uptr data_addr, const char *fmt, DataInfo DI; if (!Symbolizer::GetOrInit()->SymbolizeData(data_addr, &DI)) return; InternalScopedString data_desc; - RenderData(&data_desc, fmt, &DI, common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderData(&data_desc, fmt, &DI, + common_flags()->strip_path_prefix); internal_strncpy(out_buf, data_desc.data(), out_buf_size); out_buf[out_buf_size - 1] = 0; } diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp index 2d0eccc1602..8db321051e1 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.cpp @@ -11,25 +11,62 @@ //===----------------------------------------------------------------------===// #include "sanitizer_stacktrace_printer.h" + #include "sanitizer_file.h" +#include "sanitizer_flags.h" #include "sanitizer_fuchsia.h" namespace __sanitizer { -// sanitizer_symbolizer_markup.cpp implements these differently. -#if !SANITIZER_SYMBOLIZER_MARKUP +StackTracePrinter *StackTracePrinter::GetOrInit() { + static StackTracePrinter *stacktrace_printer; + static StaticSpinMutex init_mu; + SpinMutexLock l(&init_mu); + if (stacktrace_printer) + return stacktrace_printer; + + stacktrace_printer = + new (GetGlobalLowLevelAllocator()) FormattedStackTracePrinter(); + + CHECK(stacktrace_printer); + return stacktrace_printer; +} -static const char *StripFunctionName(const char *function, const char *prefix) { - if (!function) return nullptr; - if (!prefix) return function; - uptr prefix_len = internal_strlen(prefix); - if (0 == internal_strncmp(function, prefix, prefix_len)) - return function + prefix_len; +const char *FormattedStackTracePrinter::StripFunctionName( + const char *function) { + if (!common_flags()->demangle) + return function; + if (!function) + return nullptr; + auto try_strip = [function](const char *prefix) -> const char * { + const uptr prefix_len = internal_strlen(prefix); + if (!internal_strncmp(function, prefix, prefix_len)) + return function + prefix_len; + return nullptr; + }; + if (SANITIZER_APPLE) { + if (const char *s = try_strip("wrap_")) + return s; + } else if (SANITIZER_WINDOWS) { + if (const char *s = try_strip("__asan_wrap_")) + return s; + } else { + if (const char *s = try_strip("___interceptor_")) + return s; + if (const char *s = try_strip("__interceptor_")) + return s; + } return function; } +// sanitizer_symbolizer_markup.cpp implements these differently. +#if !SANITIZER_SYMBOLIZER_MARKUP + static const char *DemangleFunctionName(const char *function) { - if (!function) return nullptr; + if (!common_flags()->demangle) + return function; + if (!function) + return nullptr; // NetBSD uses indirection for old threading functions for historical reasons // The mangled names are internal implementation detail and should not be @@ -108,20 +145,23 @@ static void MaybeBuildIdToBuffer(const AddressInfo &info, bool PrefixSpace, InternalScopedString *buffer) { if (info.uuid_size) { if (PrefixSpace) - buffer->append(" "); - buffer->append("(BuildId: "); + buffer->AppendF(" "); + buffer->AppendF("(BuildId: "); for (uptr i = 0; i < info.uuid_size; ++i) { - buffer->append("%02x", info.uuid[i]); + buffer->AppendF("%02x", info.uuid[i]); } - buffer->append(")"); + buffer->AppendF(")"); } } static const char kDefaultFormat[] = " #%n %p %F %L"; -void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, - uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix, const char *strip_func_prefix) { +void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer, + const char *format, int frame_no, + uptr address, + const AddressInfo *info, + bool vs_style, + const char *strip_path_prefix) { // info will be null in the case where symbolization is not needed for the // given format. This ensures that the code below will get a hard failure // rather than print incorrect information in case RenderNeedsSymbolization @@ -132,56 +172,56 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, format = kDefaultFormat; for (const char *p = format; *p != '\0'; p++) { if (*p != '%') { - buffer->append("%c", *p); + buffer->AppendF("%c", *p); continue; } p++; switch (*p) { case '%': - buffer->append("%%"); + buffer->Append("%"); break; // Frame number and all fields of AddressInfo structure. case 'n': - buffer->append("%u", frame_no); + buffer->AppendF("%u", frame_no); break; case 'p': - buffer->append("0x%zx", address); + buffer->AppendF("0x%zx", address); break; case 'm': - buffer->append("%s", StripPathPrefix(info->module, strip_path_prefix)); + buffer->AppendF("%s", StripPathPrefix(info->module, strip_path_prefix)); break; case 'o': - buffer->append("0x%zx", info->module_offset); + buffer->AppendF("0x%zx", info->module_offset); break; case 'b': MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer); break; case 'f': - buffer->append("%s", DemangleFunctionName(StripFunctionName( - info->function, strip_func_prefix))); + buffer->AppendF("%s", + DemangleFunctionName(StripFunctionName(info->function))); break; case 'q': - buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown - ? info->function_offset - : 0x0); + buffer->AppendF("0x%zx", info->function_offset != AddressInfo::kUnknown + ? info->function_offset + : 0x0); break; case 's': - buffer->append("%s", StripPathPrefix(info->file, strip_path_prefix)); + buffer->AppendF("%s", StripPathPrefix(info->file, strip_path_prefix)); break; case 'l': - buffer->append("%d", info->line); + buffer->AppendF("%d", info->line); break; case 'c': - buffer->append("%d", info->column); + buffer->AppendF("%d", info->column); break; // Smarter special cases. case 'F': // Function name and offset, if file is unknown. if (info->function) { - buffer->append("in %s", DemangleFunctionName(StripFunctionName( - info->function, strip_func_prefix))); + buffer->AppendF( + "in %s", DemangleFunctionName(StripFunctionName(info->function))); if (!info->file && info->function_offset != AddressInfo::kUnknown) - buffer->append("+0x%zx", info->function_offset); + buffer->AppendF("+0x%zx", info->function_offset); } break; case 'S': @@ -198,9 +238,11 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, RenderModuleLocation(buffer, info->module, info->module_offset, info->module_arch, strip_path_prefix); +#if !SANITIZER_APPLE MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); +#endif } else { - buffer->append("(<unknown module>)"); + buffer->AppendF("(<unknown module>)"); } break; case 'M': @@ -211,9 +253,11 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, // Always strip the module name for %M. RenderModuleLocation(buffer, StripModuleName(info->module), info->module_offset, info->module_arch, ""); +#if !SANITIZER_APPLE MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); +#endif } else { - buffer->append("(%p)", (void *)address); + buffer->AppendF("(%p)", (void *)address); } break; default: @@ -224,7 +268,7 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, } } -bool RenderNeedsSymbolization(const char *format) { +bool FormattedStackTracePrinter::RenderNeedsSymbolization(const char *format) { if (0 == internal_strcmp(format, "DEFAULT")) format = kDefaultFormat; for (const char *p = format; *p != '\0'; p++) { @@ -247,26 +291,28 @@ bool RenderNeedsSymbolization(const char *format) { return false; } -void RenderData(InternalScopedString *buffer, const char *format, - const DataInfo *DI, const char *strip_path_prefix) { +void FormattedStackTracePrinter::RenderData(InternalScopedString *buffer, + const char *format, + const DataInfo *DI, + const char *strip_path_prefix) { for (const char *p = format; *p != '\0'; p++) { if (*p != '%') { - buffer->append("%c", *p); + buffer->AppendF("%c", *p); continue; } p++; switch (*p) { case '%': - buffer->append("%%"); + buffer->Append("%"); break; case 's': - buffer->append("%s", StripPathPrefix(DI->file, strip_path_prefix)); + buffer->AppendF("%s", StripPathPrefix(DI->file, strip_path_prefix)); break; case 'l': - buffer->append("%zu", DI->line); + buffer->AppendF("%zu", DI->line); break; case 'g': - buffer->append("%s", DI->name); + buffer->AppendF("%s", DI->name); break; default: Report("Unsupported specifier in stack frame format: %c (%p)!\n", *p, @@ -278,33 +324,33 @@ void RenderData(InternalScopedString *buffer, const char *format, #endif // !SANITIZER_SYMBOLIZER_MARKUP -void RenderSourceLocation(InternalScopedString *buffer, const char *file, - int line, int column, bool vs_style, - const char *strip_path_prefix) { +void FormattedStackTracePrinter::RenderSourceLocation( + InternalScopedString *buffer, const char *file, int line, int column, + bool vs_style, const char *strip_path_prefix) { if (vs_style && line > 0) { - buffer->append("%s(%d", StripPathPrefix(file, strip_path_prefix), line); + buffer->AppendF("%s(%d", StripPathPrefix(file, strip_path_prefix), line); if (column > 0) - buffer->append(",%d", column); - buffer->append(")"); + buffer->AppendF(",%d", column); + buffer->AppendF(")"); return; } - buffer->append("%s", StripPathPrefix(file, strip_path_prefix)); + buffer->AppendF("%s", StripPathPrefix(file, strip_path_prefix)); if (line > 0) { - buffer->append(":%d", line); + buffer->AppendF(":%d", line); if (column > 0) - buffer->append(":%d", column); + buffer->AppendF(":%d", column); } } -void RenderModuleLocation(InternalScopedString *buffer, const char *module, - uptr offset, ModuleArch arch, - const char *strip_path_prefix) { - buffer->append("(%s", StripPathPrefix(module, strip_path_prefix)); +void FormattedStackTracePrinter::RenderModuleLocation( + InternalScopedString *buffer, const char *module, uptr offset, + ModuleArch arch, const char *strip_path_prefix) { + buffer->AppendF("(%s", StripPathPrefix(module, strip_path_prefix)); if (arch != kModuleArchUnknown) { - buffer->append(":%s", ModuleArchToString(arch)); + buffer->AppendF(":%s", ModuleArchToString(arch)); } - buffer->append("+0x%zx)", offset); + buffer->AppendF("+0x%zx)", offset); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h index 96119b2ee9e..3e02333a8c8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_printer.h @@ -13,60 +13,102 @@ #define SANITIZER_STACKTRACE_PRINTER_H #include "sanitizer_common.h" +#include "sanitizer_internal_defs.h" #include "sanitizer_symbolizer.h" namespace __sanitizer { -// Render the contents of "info" structure, which represents the contents of -// stack frame "frame_no" and appends it to the "buffer". "format" is a -// string with placeholders, which is copied to the output with -// placeholders substituted with the contents of "info". For example, -// format string -// " frame %n: function %F at %S" -// will be turned into -// " frame 10: function foo::bar() at my/file.cc:10" -// You may additionally pass "strip_path_prefix" to strip prefixes of paths to -// source files and modules, and "strip_func_prefix" to strip prefixes of -// function names. -// Here's the full list of available placeholders: -// %% - represents a '%' character; -// %n - frame number (copy of frame_no); -// %p - PC in hex format; -// %m - path to module (binary or shared object); -// %o - offset in the module in hex format; -// %f - function name; -// %q - offset in the function in hex format (*if available*); -// %s - path to source file; -// %l - line in the source file; -// %c - column in the source file; -// %F - if function is known to be <foo>, prints "in <foo>", possibly -// followed by the offset in this function, but only if source file -// is unknown; -// %S - prints file/line/column information; -// %L - prints location information: file/line/column, if it is known, or -// module+offset if it is known, or (<unknown module>) string. -// %M - prints module basename and offset, if it is known, or PC. -void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, - uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix = "", - const char *strip_func_prefix = ""); - -bool RenderNeedsSymbolization(const char *format); - -void RenderSourceLocation(InternalScopedString *buffer, const char *file, - int line, int column, bool vs_style, - const char *strip_path_prefix); - -void RenderModuleLocation(InternalScopedString *buffer, const char *module, - uptr offset, ModuleArch arch, - const char *strip_path_prefix); - -// Same as RenderFrame, but for data section (global variables). -// Accepts %s, %l from above. -// Also accepts: -// %g - name of the global variable. -void RenderData(InternalScopedString *buffer, const char *format, - const DataInfo *DI, const char *strip_path_prefix = ""); +// StacktracePrinter is an interface that is implemented by +// classes that can perform rendering of the different parts +// of a stacktrace. +class StackTracePrinter { + public: + static StackTracePrinter *GetOrInit(); + + virtual const char *StripFunctionName(const char *function) = 0; + + virtual void RenderFrame(InternalScopedString *buffer, const char *format, + int frame_no, uptr address, const AddressInfo *info, + bool vs_style, + const char *strip_path_prefix = "") = 0; + + virtual bool RenderNeedsSymbolization(const char *format) = 0; + + virtual void RenderSourceLocation(InternalScopedString *buffer, + const char *file, int line, int column, + bool vs_style, + const char *strip_path_prefix) = 0; + + virtual void RenderModuleLocation(InternalScopedString *buffer, + const char *module, uptr offset, + ModuleArch arch, + const char *strip_path_prefix) = 0; + virtual void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, + const char *strip_path_prefix = "") = 0; + + protected: + ~StackTracePrinter() {} +}; + +class FormattedStackTracePrinter : public StackTracePrinter { + public: + // Strip interceptor prefixes from function name. + const char *StripFunctionName(const char *function) override; + + // Render the contents of "info" structure, which represents the contents of + // stack frame "frame_no" and appends it to the "buffer". "format" is a + // string with placeholders, which is copied to the output with + // placeholders substituted with the contents of "info". For example, + // format string + // " frame %n: function %F at %S" + // will be turned into + // " frame 10: function foo::bar() at my/file.cc:10" + // You may additionally pass "strip_path_prefix" to strip prefixes of paths to + // source files and modules. + // Here's the full list of available placeholders: + // %% - represents a '%' character; + // %n - frame number (copy of frame_no); + // %p - PC in hex format; + // %m - path to module (binary or shared object); + // %o - offset in the module in hex format; + // %f - function name; + // %q - offset in the function in hex format (*if available*); + // %s - path to source file; + // %l - line in the source file; + // %c - column in the source file; + // %F - if function is known to be <foo>, prints "in <foo>", possibly + // followed by the offset in this function, but only if source file + // is unknown; + // %S - prints file/line/column information; + // %L - prints location information: file/line/column, if it is known, or + // module+offset if it is known, or (<unknown module>) string. + // %M - prints module basename and offset, if it is known, or PC. + void RenderFrame(InternalScopedString *buffer, const char *format, + int frame_no, uptr address, const AddressInfo *info, + bool vs_style, const char *strip_path_prefix = "") override; + + bool RenderNeedsSymbolization(const char *format) override; + + void RenderSourceLocation(InternalScopedString *buffer, const char *file, + int line, int column, bool vs_style, + const char *strip_path_prefix) override; + + void RenderModuleLocation(InternalScopedString *buffer, const char *module, + uptr offset, ModuleArch arch, + const char *strip_path_prefix) override; + + // Same as RenderFrame, but for data section (global variables). + // Accepts %s, %l from above. + // Also accepts: + // %g - name of the global variable. + void RenderData(InternalScopedString *buffer, const char *format, + const DataInfo *DI, + const char *strip_path_prefix = "") override; + + protected: + ~FormattedStackTracePrinter() {} +}; } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp index 13b90ce9bf5..25c4af70856 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp @@ -565,7 +565,7 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP( constexpr uptr uptr_sz = sizeof(uptr); int pterrno; #ifdef ARCH_IOVEC_FOR_GETREGSET - auto append = [&](uptr regset) { + auto AppendF = [&](uptr regset) { uptr size = buffer->size(); // NT_X86_XSTATE requires 64bit alignment. uptr size_up = RoundUpTo(size, 8 / uptr_sz); @@ -596,11 +596,11 @@ PtraceRegistersStatus SuspendedThreadsListLinux::GetRegistersAndSP( }; buffer->clear(); - bool fail = !append(NT_PRSTATUS); + bool fail = !AppendF(NT_PRSTATUS); if (!fail) { // Accept the first available and do not report errors. for (uptr regs : kExtraRegs) - if (regs && append(regs)) + if (regs && AppendF(regs)) break; } #else diff --git a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp index 3ebeac52280..81361646765 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stoptheworld_mac.cpp @@ -154,12 +154,10 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP( ®_count); if (err != KERN_SUCCESS) { VReport(1, "Error - unable to get registers for a thread\n"); - // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid, - // or the thread does not exist. The other possible error case, // MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's // still safe to proceed. - return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL - : REGISTERS_UNAVAILABLE; + return err == MIG_ARRAY_TOO_LARGE ? REGISTERS_UNAVAILABLE + : REGISTERS_UNAVAILABLE_FATAL; } buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr)); diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cpp index d3cffaa6eef..519f768f896 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.cpp @@ -10,6 +10,8 @@ // run-time libraries. //===----------------------------------------------------------------------===// +#include <errno.h> + #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_internal_defs.h" @@ -128,7 +130,7 @@ Symbolizer::Symbolizer(IntrusiveList<SymbolizerTool> tools) start_hook_(0), end_hook_(0) {} Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym) - : sym_(sym) { + : sym_(sym), errno_(errno) { if (sym_->start_hook_) sym_->start_hook_(); } @@ -136,6 +138,7 @@ Symbolizer::SymbolizerScope::SymbolizerScope(const Symbolizer *sym) Symbolizer::SymbolizerScope::~SymbolizerScope() { if (sym_->end_hook_) sym_->end_hook_(); + errno = errno_; } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h index bad4761e345..7fb7928dce0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer.h @@ -136,7 +136,7 @@ class Symbolizer final { // Release internal caches (if any). void Flush(); - // Attempts to demangle the provided C++ mangled name. + // Attempts to demangle the provided C++ mangled name. Never returns nullptr. const char *Demangle(const char *name); // Allow user to install hooks that would be called before/after Symbolizer @@ -187,7 +187,7 @@ class Symbolizer final { // If stale, need to reload the modules before looking up addresses. bool modules_fresh_; - // Platform-specific default demangler, must not return nullptr. + // Platform-specific default demangler, returns nullptr on failure. const char *PlatformDemangle(const char *name); static Symbolizer *symbolizer_; @@ -212,6 +212,7 @@ class Symbolizer final { ~SymbolizerScope(); private: const Symbolizer *sym_; + int errno_; // Backup errno in case symbolizer change the value. }; }; diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h index 3ec4d80105a..2345aee9855 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_internal.h @@ -160,6 +160,15 @@ void ParseSymbolizePCOutput(const char *str, SymbolizedStack *res); // Used by LLVMSymbolizer and InternalSymbolizer. void ParseSymbolizeDataOutput(const char *str, DataInfo *info); +// Parses repeated strings in the following format: +// <function_name> +// <var_name> +// <file_name>:<line_number>[:<column_number>] +// [<frame_offset>|??] [<size>|??] [<tag_offset>|??] +// Used by LLVMSymbolizer and InternalSymbolizer. +void ParseSymbolizeFrameOutput(const char *str, + InternalMmapVector<LocalInfo> *locals); + } // namespace __sanitizer #endif // SANITIZER_SYMBOLIZER_INTERNAL_H diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp index cc02c77bccd..d78dab93487 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp @@ -199,7 +199,7 @@ static char *DemangleAlloc(const char *name, bool always_alloc) { #endif if (always_alloc) return internal_strdup(name); - return 0; + return nullptr; } const char *LibbacktraceSymbolizer::Demangle(const char *name) { diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp index a6f82ced203..81141023386 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_libcdep.cpp @@ -117,7 +117,7 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { return true; } } - return true; + return false; } bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { @@ -133,7 +133,7 @@ bool Symbolizer::SymbolizeFrame(uptr addr, FrameInfo *info) { return true; } } - return true; + return false; } bool Symbolizer::GetModuleNameAndOffsetForPC(uptr pc, const char **module_name, @@ -159,13 +159,16 @@ void Symbolizer::Flush() { } const char *Symbolizer::Demangle(const char *name) { + CHECK(name); Lock l(&mu_); for (auto &tool : tools_) { SymbolizerScope sym_scope(this); if (const char *demangled = tool.Demangle(name)) return demangled; } - return PlatformDemangle(name); + if (const char *demangled = PlatformDemangle(name)) + return demangled; + return name; } bool Symbolizer::FindModuleNameAndOffsetForAddress(uptr address, @@ -382,8 +385,8 @@ void ParseSymbolizeDataOutput(const char *str, DataInfo *info) { str = ExtractUptr(str, "\n", &info->line); } -static void ParseSymbolizeFrameOutput(const char *str, - InternalMmapVector<LocalInfo> *locals) { +void ParseSymbolizeFrameOutput(const char *str, + InternalMmapVector<LocalInfo> *locals) { if (internal_strncmp(str, "??", 2) == 0) return; diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp index a9c958b2d10..f1cc0b5e1e8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_mac.cpp @@ -42,7 +42,8 @@ bool DlAddrSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { } const char *demangled = DemangleSwiftAndCXX(info.dli_sname); - if (!demangled) return false; + if (!demangled) + demangled = info.dli_sname; stack->info.function = internal_strdup(demangled); return true; } @@ -52,6 +53,8 @@ bool DlAddrSymbolizer::SymbolizeData(uptr addr, DataInfo *datainfo) { int result = dladdr((const void *)addr, &info); if (!result) return false; const char *demangled = DemangleSwiftAndCXX(info.dli_sname); + if (!demangled) + demangled = info.dli_sname; datainfo->name = internal_strdup(demangled); datainfo->start = (uptr)info.dli_saddr; return true; diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp index 1ec0c5cad7a..d1b0f46004e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp @@ -22,6 +22,7 @@ # include <unwind.h> # include "sanitizer_stacktrace.h" +# include "sanitizer_stacktrace_printer.h" # include "sanitizer_symbolizer.h" namespace __sanitizer { @@ -81,19 +82,26 @@ bool Symbolizer::SymbolizeData(uptr addr, DataInfo *info) { } // We ignore the format argument to __sanitizer_symbolize_global. -void RenderData(InternalScopedString *buffer, const char *format, - const DataInfo *DI, const char *strip_path_prefix) { - buffer->append(kFormatData, DI->start); +void FormattedStackTracePrinter::RenderData(InternalScopedString *buffer, + const char *format, + const DataInfo *DI, + const char *strip_path_prefix) { + buffer->AppendF(kFormatData, DI->start); } -bool RenderNeedsSymbolization(const char *format) { return false; } +bool FormattedStackTracePrinter::RenderNeedsSymbolization(const char *format) { + return false; +} // We don't support the stack_trace_format flag at all. -void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, - uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix, const char *strip_func_prefix) { +void FormattedStackTracePrinter::RenderFrame(InternalScopedString *buffer, + const char *format, int frame_no, + uptr address, + const AddressInfo *info, + bool vs_style, + const char *strip_path_prefix) { CHECK(!RenderNeedsSymbolization(format)); - buffer->append(kFormatFrame, frame_no, address); + buffer->AppendF(kFormatFrame, frame_no, address); } Symbolizer *Symbolizer::PlatformInit() { diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index 1a5e38faea8..d92349c04ff 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -56,7 +56,7 @@ const char *DemangleCXXABI(const char *name) { __cxxabiv1::__cxa_demangle(name, 0, 0, 0)) return demangled_name; - return name; + return nullptr; } // As of now, there are no headers for the Swift runtime. Once they are @@ -324,9 +324,12 @@ __sanitizer_symbolize_code(const char *ModuleName, u64 ModuleOffset, SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool __sanitizer_symbolize_data(const char *ModuleName, u64 ModuleOffset, char *Buffer, int MaxLength); +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool +__sanitizer_symbolize_frame(const char *ModuleName, u64 ModuleOffset, + char *Buffer, int MaxLength); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void __sanitizer_symbolize_flush(); -SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int +SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool __sanitizer_symbolize_demangle(const char *Name, char *Buffer, int MaxLength); SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE bool __sanitizer_symbolize_set_demangle(bool Demangle); @@ -337,19 +340,20 @@ __sanitizer_symbolize_set_inline_frames(bool InlineFrames); class InternalSymbolizer final : public SymbolizerTool { public: static InternalSymbolizer *get(LowLevelAllocator *alloc) { - if (__sanitizer_symbolize_set_demangle) + if (&__sanitizer_symbolize_set_demangle) CHECK(__sanitizer_symbolize_set_demangle(common_flags()->demangle)); - if (__sanitizer_symbolize_set_inline_frames) + if (&__sanitizer_symbolize_set_inline_frames) CHECK(__sanitizer_symbolize_set_inline_frames( common_flags()->symbolize_inline_frames)); - if (__sanitizer_symbolize_code && __sanitizer_symbolize_data) + // These are essential, we don't have InternalSymbolizer without them. + if (&__sanitizer_symbolize_code && &__sanitizer_symbolize_data) return new (*alloc) InternalSymbolizer(); return 0; } bool SymbolizePC(uptr addr, SymbolizedStack *stack) override { bool result = __sanitizer_symbolize_code( - stack->info.module, stack->info.module_offset, buffer_, kBufferSize); + stack->info.module, stack->info.module_offset, buffer_, sizeof(buffer_)); if (result) ParseSymbolizePCOutput(buffer_, stack); return result; @@ -357,7 +361,7 @@ class InternalSymbolizer final : public SymbolizerTool { bool SymbolizeData(uptr addr, DataInfo *info) override { bool result = __sanitizer_symbolize_data(info->module, info->module_offset, - buffer_, kBufferSize); + buffer_, sizeof(buffer_)); if (result) { ParseSymbolizeDataOutput(buffer_, info); info->start += (addr - info->module_offset); // Add the base address. @@ -365,34 +369,35 @@ class InternalSymbolizer final : public SymbolizerTool { return result; } + bool SymbolizeFrame(uptr addr, FrameInfo *info) override { + if (&__sanitizer_symbolize_frame == nullptr) + return false; + bool result = __sanitizer_symbolize_frame(info->module, info->module_offset, + buffer_, sizeof(buffer_)); + if (result) + ParseSymbolizeFrameOutput(buffer_, &info->locals); + return result; + } + void Flush() override { - if (__sanitizer_symbolize_flush) + if (&__sanitizer_symbolize_flush) __sanitizer_symbolize_flush(); } const char *Demangle(const char *name) override { - if (__sanitizer_symbolize_demangle) { - for (uptr res_length = 1024; - res_length <= InternalSizeClassMap::kMaxSize;) { - char *res_buff = static_cast<char *>(InternalAlloc(res_length)); - uptr req_length = - __sanitizer_symbolize_demangle(name, res_buff, res_length); - if (req_length > res_length) { - res_length = req_length + 1; - InternalFree(res_buff); - continue; - } - return res_buff; - } + if (&__sanitizer_symbolize_demangle && + __sanitizer_symbolize_demangle(name, buffer_, sizeof(buffer_))) { + char *res_buff = nullptr; + ExtractToken(buffer_, "", &res_buff); + return res_buff; } - return name; + return nullptr; } private: InternalSymbolizer() {} - static const int kBufferSize = 16 * 1024; - char buffer_[kBufferSize]; + char buffer_[16 * 1024]; }; # else // SANITIZER_SUPPORTS_WEAK_HOOKS diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp index 73915715c5b..3e4417ae3f5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -32,10 +32,10 @@ void ReportErrorSummary(const char *error_type, const AddressInfo &info, const char *alt_tool_name) { if (!common_flags()->print_summary) return; InternalScopedString buff; - buff.append("%s ", error_type); - RenderFrame(&buff, "%L %F", 0, info.address, &info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); + buff.AppendF("%s ", error_type); + StackTracePrinter::GetOrInit()->RenderFrame( + &buff, "%L %F", 0, info.address, &info, + common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); ReportErrorSummary(buff.data(), alt_tool_name); } #endif @@ -148,22 +148,22 @@ static void MaybeReportNonExecRegion(uptr pc) { static void PrintMemoryByte(InternalScopedString *str, const char *before, u8 byte) { SanitizerCommonDecorator d; - str->append("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15, - d.Default()); + str->AppendF("%s%s%x%x%s ", before, d.MemoryByte(), byte >> 4, byte & 15, + d.Default()); } static void MaybeDumpInstructionBytes(uptr pc) { if (!common_flags()->dump_instruction_bytes || (pc < GetPageSizeCached())) return; InternalScopedString str; - str.append("First 16 instruction bytes at pc: "); + str.AppendF("First 16 instruction bytes at pc: "); if (IsAccessibleMemoryRange(pc, 16)) { for (int i = 0; i < 16; ++i) { PrintMemoryByte(&str, "", ((u8 *)pc)[i]); } - str.append("\n"); + str.AppendF("\n"); } else { - str.append("unaccessible\n"); + str.AppendF("unaccessible\n"); } Report("%s", str.data()); } diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cpp index 2a7137fe1f8..aae3e76ea22 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_win.cpp @@ -175,9 +175,7 @@ const char *WinSymbolizerTool::Demangle(const char *name) { return name; } -const char *Symbolizer::PlatformDemangle(const char *name) { - return name; -} +const char *Symbolizer::PlatformDemangle(const char *name) { return nullptr; } namespace { struct ScopedHandle { @@ -233,7 +231,7 @@ bool SymbolizerProcess::StartSymbolizerSubprocess() { CHECK(!internal_strchr(arg, '"') && "quotes in args unsupported"); CHECK(arglen > 0 && arg[arglen - 1] != '\\' && "args ending in backslash and empty args unsupported"); - command_line.append("\"%s\" ", arg); + command_line.AppendF("\"%s\" ", arg); } VReport(3, "Launching symbolizer command: %s\n", command_line.data()); @@ -292,15 +290,15 @@ static void ChooseSymbolizerTools(IntrusiveList<SymbolizerTool> *list, const char *path = user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe"); if (path) { - VReport(2, "Using llvm-symbolizer at %spath: %s\n", - user_path ? "user-specified " : "", path); - list->push_back(new(*allocator) LLVMSymbolizer(path, allocator)); - } else { if (user_path && user_path[0] == '\0') { VReport(2, "External symbolizer is explicitly disabled.\n"); } else { - VReport(2, "External symbolizer is not present.\n"); + VReport(2, "Using llvm-symbolizer at %spath: %s\n", + user_path ? "user-specified " : "", path); + list->push_back(new (*allocator) LLVMSymbolizer(path, allocator)); } + } else { + VReport(2, "External symbolizer is not present.\n"); } // Add the dbghelp based symbolizer. diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.cpp b/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.cpp new file mode 100644 index 00000000000..bddb2852140 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.cpp @@ -0,0 +1,94 @@ +//===-- sanitizer_thread_arg_retval.cpp -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizer tools. +// +// Tracks thread arguments and return value for leak checking. +//===----------------------------------------------------------------------===// + +#include "sanitizer_thread_arg_retval.h" + +#include "sanitizer_placement_new.h" + +namespace __sanitizer { + +void ThreadArgRetval::CreateLocked(uptr thread, bool detached, + const Args& args) { + CheckLocked(); + Data& t = data_[thread]; + t = {}; + t.gen = gen_++; + t.detached = detached; + t.args = args; +} + +ThreadArgRetval::Args ThreadArgRetval::GetArgs(uptr thread) const { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + CHECK(t); + if (t->second.done) + return {}; + return t->second.args; +} + +void ThreadArgRetval::Finish(uptr thread, void* retval) { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + if (!t) + return; + if (t->second.detached) { + // Retval of detached thread connot be retrieved. + data_.erase(t); + return; + } + t->second.done = true; + t->second.args.arg_retval = retval; +} + +u32 ThreadArgRetval::BeforeJoin(uptr thread) const { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + CHECK(t); + CHECK(!t->second.detached); + return t->second.gen; +} + +void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + if (!t || gen != t->second.gen) { + // Thread was reused and erased by any other event. + return; + } + CHECK(!t->second.detached); + data_.erase(t); +} + +void ThreadArgRetval::DetachLocked(uptr thread) { + CheckLocked(); + auto t = data_.find(thread); + CHECK(t); + CHECK(!t->second.detached); + if (t->second.done) { + // We can't retrive retval after detached thread finished. + data_.erase(t); + return; + } + t->second.detached = true; +} + +void ThreadArgRetval::GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs) { + CheckLocked(); + CHECK(ptrs); + data_.forEach([&](DenseMap<uptr, Data>::value_type& kv) -> bool { + ptrs->push_back((uptr)kv.second.args.arg_retval); + return true; + }); +} + +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.h b/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.h new file mode 100644 index 00000000000..c77021beb67 --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_thread_arg_retval.h @@ -0,0 +1,116 @@ +//===-- sanitizer_thread_arg_retval.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizer tools. +// +// Tracks thread arguments and return value for leak checking. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_THREAD_ARG_RETVAL_H +#define SANITIZER_THREAD_ARG_RETVAL_H + +#include "sanitizer_common.h" +#include "sanitizer_dense_map.h" +#include "sanitizer_list.h" +#include "sanitizer_mutex.h" + +namespace __sanitizer { + +// Primary goal of the class is to keep alive arg and retval pointer for leak +// checking. However it can be used to pass those pointer into wrappers used by +// interceptors. The difference from ThreadRegistry/ThreadList is that this +// class keeps data up to the detach or join, as exited thread still can be +// joined to retrive retval. ThreadRegistry/ThreadList can discard exited +// threads immediately. +class SANITIZER_MUTEX ThreadArgRetval { + public: + struct Args { + void* (*routine)(void*); + void* arg_retval; // Either arg or retval. + }; + void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); } + void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); } + void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); } + + // Wraps pthread_create or similar. We need to keep object locked, to + // prevent child thread from proceeding without thread handle. + template <typename CreateFn /* returns thread id on success, or 0 */> + void Create(bool detached, const Args& args, const CreateFn& fn) { + // No need to track detached threads with no args, but we will to do as it's + // not expensive and less edge-cases. + __sanitizer::Lock lock(&mtx_); + if (uptr thread = fn()) + CreateLocked(thread, detached, args); + } + + // Returns thread arg and routine. + Args GetArgs(uptr thread) const; + + // Mark thread as done and stores retval or remove if detached. Should be + // called by the thread. + void Finish(uptr thread, void* retval); + + // Mark thread as detached or remove if done. + template <typename DetachFn /* returns true on success */> + void Detach(uptr thread, const DetachFn& fn) { + // Lock to prevent re-use of the thread between fn() and DetachLocked() + // calls. + __sanitizer::Lock lock(&mtx_); + if (fn()) + DetachLocked(thread); + } + + // Joins the thread. + template <typename JoinFn /* returns true on success */> + void Join(uptr thread, const JoinFn& fn) { + // Remember internal id of the thread to prevent re-use of the thread + // between fn() and AfterJoin() calls. Locking JoinFn, like in + // Detach(), implementation can cause deadlock. + auto gen = BeforeJoin(thread); + if (fn()) + AfterJoin(thread, gen); + } + + // Returns all arg and retval which are considered alive. + void GetAllPtrsLocked(InternalMmapVector<uptr>* ptrs); + + uptr size() const { + __sanitizer::Lock lock(&mtx_); + return data_.size(); + } + + // FIXME: Add fork support. Expected users of the class are sloppy with forks + // anyway. We likely should lock/unlock the object to avoid deadlocks, and + // erase all but the current threads, so we can detect leaked arg or retval in + // child process. + + // FIXME: Add cancelation support. Now if a thread was canceled, the class + // will keep pointers alive forever, missing leaks caused by cancelation. + + private: + struct Data { + Args args; + u32 gen; // Avoid collision if thread id re-used. + bool detached; + bool done; + }; + + void CreateLocked(uptr thread, bool detached, const Args& args); + u32 BeforeJoin(uptr thread) const; + void AfterJoin(uptr thread, u32 gen); + void DetachLocked(uptr thread); + + mutable Mutex mtx_; + + DenseMap<uptr, Data> data_; + u32 gen_ = 0; +}; + +} // namespace __sanitizer + +#endif // SANITIZER_THREAD_ARG_RETVAL_H diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_win.cpp index 1c9b2dd8a5e..06e496523ee 100644 --- a/libsanitizer/sanitizer_common/sanitizer_win.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_win.cpp @@ -362,6 +362,11 @@ bool MprotectReadOnly(uptr addr, uptr size) { return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection); } +bool MprotectReadWrite(uptr addr, uptr size) { + DWORD old_protection; + return VirtualProtect((LPVOID)addr, size, PAGE_READWRITE, &old_protection); +} + void ReleaseMemoryPagesToOS(uptr beg, uptr end) { uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()), end_aligned = RoundDownTo(end, GetPageSizeCached()); diff --git a/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h index 48c73c4c98a..639d91a2eda 100644 --- a/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h +++ b/libsanitizer/sanitizer_common/sanitizer_win_dll_thunk.h @@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init(); // which isn't a big deal. #define INTERCEPT_LIBRARY_FUNCTION(name) \ extern "C" void name(); \ - INTERCEPT_OR_DIE(WRAPPER_NAME(name), name) + INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name) // Use these macros for functions that could be called before __dll_thunk_init() // is executed and don't lead to errors if defined (free, malloc, etc). diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am index 01290b0313d..cb8bf2e705e 100644 --- a/libsanitizer/tsan/Makefile.am +++ b/libsanitizer/tsan/Makefile.am @@ -22,6 +22,7 @@ tsan_files = \ tsan_ignoreset.cpp \ tsan_interceptors_posix.cpp \ tsan_interceptors_mac.cpp \ + tsan_interceptors_memintrinsics.cpp \ tsan_interface_ann.cpp \ tsan_interface_atomic.cpp \ tsan_interface.cpp \ @@ -49,7 +50,7 @@ tsan_files = \ tsan_vector_clock.cpp libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S tsan_rtl_riscv64.S libtsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) libtsan_la_DEPENDENCIES = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) if LIBBACKTRACE_SUPPORTED diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in index 95011584bcb..fc14a42c082 100644 --- a/libsanitizer/tsan/Makefile.in +++ b/libsanitizer/tsan/Makefile.in @@ -148,10 +148,10 @@ LTLIBRARIES = $(toolexeclib_LTLIBRARIES) am__DEPENDENCIES_1 = am__objects_1 = tsan_debugging.lo tsan_external.lo tsan_fd.lo \ tsan_flags.lo tsan_ignoreset.lo tsan_interceptors_posix.lo \ - tsan_interceptors_mac.lo tsan_interface_ann.lo \ - tsan_interface_atomic.lo tsan_interface.lo \ - tsan_interface_java.lo tsan_malloc_mac.lo tsan_md5.lo \ - tsan_mman.lo tsan_mutexset.lo tsan_new_delete.lo \ + tsan_interceptors_mac.lo tsan_interceptors_memintrinsics.lo \ + tsan_interface_ann.lo tsan_interface_atomic.lo \ + tsan_interface.lo tsan_interface_java.lo tsan_malloc_mac.lo \ + tsan_md5.lo tsan_mman.lo tsan_mutexset.lo tsan_new_delete.lo \ tsan_platform_linux.lo tsan_platform_mac.lo \ tsan_platform_posix.lo tsan_platform_windows.lo tsan_report.lo \ tsan_rtl.lo tsan_rtl_access.lo tsan_rtl_mutex.lo \ @@ -427,6 +427,7 @@ tsan_files = \ tsan_ignoreset.cpp \ tsan_interceptors_posix.cpp \ tsan_interceptors_mac.cpp \ + tsan_interceptors_memintrinsics.cpp \ tsan_interface_ann.cpp \ tsan_interface_atomic.cpp \ tsan_interface.cpp \ @@ -454,7 +455,7 @@ tsan_files = \ tsan_vector_clock.cpp libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S tsan_rtl_riscv64.S libtsan_la_LIBADD = \ $(top_builddir)/sanitizer_common/libsanitizer_common.la \ $(top_builddir)/interception/libinterception.la \ @@ -592,6 +593,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_flags.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_ignoreset.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interceptors_mac.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interceptors_memintrinsics.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interceptors_posix.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interface.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_interface_ann.Plo@am__quote@ @@ -616,6 +618,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_ppc64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_proc.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_report.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_riscv64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_s390x.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_thread.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stack_trace.Plo@am__quote@ diff --git a/libsanitizer/tsan/tsan_debugging.cpp b/libsanitizer/tsan/tsan_debugging.cpp index 1e61c31c5a9..41fa293dbaa 100644 --- a/libsanitizer/tsan/tsan_debugging.cpp +++ b/libsanitizer/tsan/tsan_debugging.cpp @@ -35,7 +35,9 @@ static const char *ReportTypeDescription(ReportType typ) { case ReportTypeSignalUnsafe: return "signal-unsafe-call"; case ReportTypeErrnoInSignal: return "errno-in-signal-handler"; case ReportTypeDeadlock: return "lock-order-inversion"; - // No default case so compiler warns us if we miss one + case ReportTypeMutexHeldWrongContext: + return "mutex-held-in-wrong-context"; + // No default case so compiler warns us if we miss one } UNREACHABLE("missing case"); } diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h index 60fbc58f988..a357a870fdf 100644 --- a/libsanitizer/tsan/tsan_interceptors.h +++ b/libsanitizer/tsan/tsan_interceptors.h @@ -29,6 +29,11 @@ class ScopedInterceptor { void EnableIgnoresImpl(); }; +struct TsanInterceptorContext { + ThreadState *thr; + const uptr pc; +}; + LibIgnore *libignore(); #if !SANITIZER_GO @@ -82,7 +87,7 @@ inline bool MustIgnoreInterceptor(ThreadState *thr) { #if SANITIZER_FREEBSD # define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) \ TSAN_INTERCEPTOR(ret, _pthread_##func, __VA_ARGS__) \ - ALIAS(WRAPPER_NAME(pthread_##func)); + ALIAS(WRAP(pthread_##func)); #else # define TSAN_INTERCEPTOR_FREEBSD_ALIAS(ret, func, ...) #endif @@ -90,17 +95,38 @@ inline bool MustIgnoreInterceptor(ThreadState *thr) { #if SANITIZER_NETBSD # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) \ TSAN_INTERCEPTOR(ret, __libc_##func, __VA_ARGS__) \ - ALIAS(WRAPPER_NAME(pthread_##func)); + ALIAS(WRAP(pthread_##func)); # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) \ TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ - ALIAS(WRAPPER_NAME(pthread_##func)); + ALIAS(WRAP(pthread_##func)); # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) \ TSAN_INTERCEPTOR(ret, __libc_thr_##func, __VA_ARGS__) \ - ALIAS(WRAPPER_NAME(pthread_##func2)); + ALIAS(WRAP(pthread_##func2)); #else # define TSAN_INTERCEPTOR_NETBSD_ALIAS(ret, func, ...) # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(ret, func, ...) # define TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(ret, func, func2, ...) #endif +#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) + +#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ + (!cur_thread_init()->is_inited) + +#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ + ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ + true) + +#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ + MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ + ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ + false) + +#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ + SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ + TsanInterceptorContext _ctx = {thr, pc}; \ + ctx = (void *)&_ctx; \ + (void)ctx; + #endif // TSAN_INTERCEPTORS_H diff --git a/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp b/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp index 88d5f0a4811..2104fe7fd05 100644 --- a/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp +++ b/libsanitizer/tsan/tsan_interceptors_libdispatch.cpp @@ -558,7 +558,7 @@ TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, } DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) -DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) +DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, size_t size, dispatch_queue_t q, dispatch_block_t destructor) { diff --git a/libsanitizer/tsan/tsan_interceptors_memintrinsics.cpp b/libsanitizer/tsan/tsan_interceptors_memintrinsics.cpp new file mode 100644 index 00000000000..c8b6b2ef194 --- /dev/null +++ b/libsanitizer/tsan/tsan_interceptors_memintrinsics.cpp @@ -0,0 +1,43 @@ +//===-- tsan_interceptors_posix.cpp ---------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of ThreadSanitizer (TSan), a race detector. +// +//===----------------------------------------------------------------------===// + +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + +#include "tsan_interceptors.h" +#include "tsan_interface.h" + +using namespace __tsan; + +#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc" + +extern "C" { + +void *__tsan_memcpy(void *dst, const void *src, uptr size) { + void *ctx; +#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); +#else + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +#endif +} + +void *__tsan_memset(void *dst, int c, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size); +} + +void *__tsan_memmove(void *dst, const void *src, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +} + +} // extern "C" diff --git a/libsanitizer/tsan/tsan_interceptors_posix.cpp b/libsanitizer/tsan/tsan_interceptors_posix.cpp index 6ac6ac6a7fb..80f86ca98ed 100644 --- a/libsanitizer/tsan/tsan_interceptors_posix.cpp +++ b/libsanitizer/tsan/tsan_interceptors_posix.cpp @@ -35,6 +35,9 @@ using namespace __tsan; +DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size) +DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size) + #if SANITIZER_FREEBSD || SANITIZER_APPLE #define stdout __stdoutp #define stderr __stderrp @@ -78,6 +81,8 @@ struct ucontext_t { #define PTHREAD_ABI_BASE "GLIBC_2.17" #elif SANITIZER_LOONGARCH64 #define PTHREAD_ABI_BASE "GLIBC_2.36" +#elif SANITIZER_RISCV64 +# define PTHREAD_ABI_BASE "GLIBC_2.27" #endif extern "C" int pthread_attr_init(void *attr); @@ -128,7 +133,9 @@ const int SIGSYS = 12; const int SIGBUS = 7; const int SIGSYS = 31; #endif +#if SANITIZER_HAS_SIGINFO const int SI_TIMER = -2; +#endif void *const MAP_FAILED = (void*)-1; #if SANITIZER_NETBSD const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; @@ -156,9 +163,6 @@ const int SA_SIGINFO = 4; const int SIG_SETMASK = 2; #endif -#define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ - (!cur_thread_init()->is_inited) - namespace __tsan { struct SignalDesc { bool armed; @@ -592,58 +596,27 @@ TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #define sigsetjmp_symname sigsetjmp #endif -#define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x -#define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) -#define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) -#define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) - -#define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname) -#define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname) - -// Not called. Merely to satisfy TSAN_INTERCEPT(). -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int TSAN_INTERCEPTOR_SETJMP(void *env); -extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { - CHECK(0); - return 0; -} - -// FIXME: any reason to have a separate declaration? -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor__setjmp(void *env); -extern "C" int __interceptor__setjmp(void *env) { - CHECK(0); - return 0; -} - -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int TSAN_INTERCEPTOR_SIGSETJMP(void *env); -extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { - CHECK(0); - return 0; -} - -#if !SANITIZER_NETBSD -extern "C" SANITIZER_INTERFACE_ATTRIBUTE -int __interceptor___sigsetjmp(void *env); -extern "C" int __interceptor___sigsetjmp(void *env) { - CHECK(0); - return 0; -} -#endif - -extern "C" int setjmp_symname(void *env); -extern "C" int _setjmp(void *env); -extern "C" int sigsetjmp_symname(void *env); -#if !SANITIZER_NETBSD -extern "C" int __sigsetjmp(void *env); -#endif DEFINE_REAL(int, setjmp_symname, void *env) DEFINE_REAL(int, _setjmp, void *env) DEFINE_REAL(int, sigsetjmp_symname, void *env) #if !SANITIZER_NETBSD DEFINE_REAL(int, __sigsetjmp, void *env) #endif + +// The real interceptor for setjmp is special, and implemented in pure asm. We +// just need to initialize the REAL functions so that they can be used in asm. +static void InitializeSetjmpInterceptors() { + // We can not use TSAN_INTERCEPT to get setjmp addr, because it does &setjmp and + // setjmp is not present in some versions of libc. + using __interception::InterceptFunction; + InterceptFunction(SANITIZER_STRINGIFY(setjmp_symname), (uptr*)&REAL(setjmp_symname), 0, 0); + InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); + InterceptFunction(SANITIZER_STRINGIFY(sigsetjmp_symname), (uptr*)&REAL(sigsetjmp_symname), 0, + 0); +#if !SANITIZER_NETBSD + InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); +#endif +} #endif // SANITIZER_APPLE #if SANITIZER_NETBSD @@ -824,10 +797,11 @@ static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, return res; } -TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { - SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); +template <class Munmap> +static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap, + void *addr, SIZE_T sz) { UnmapShadow(thr, (uptr)addr, sz); - int res = REAL(munmap)(addr, sz); + int res = real_munmap(addr, sz); return res; } @@ -2420,11 +2394,6 @@ static int OnExit(ThreadState *thr) { return status; } -struct TsanInterceptorContext { - ThreadState *thr; - const uptr pc; -}; - #if !SANITIZER_APPLE static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { @@ -2446,28 +2415,11 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK -#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ INTERCEPT_FUNCTION_VER(name, ver) #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name)) -#define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ - MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ - ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ - true) - -#define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ - MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ - ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ - false) - -#define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ - SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ - TsanInterceptorContext _ctx = {thr, pc}; \ - ctx = (void *)&_ctx; \ - (void)ctx; - #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, pc}; \ @@ -2555,6 +2507,11 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, off); \ } while (false) +#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \ + do { \ + return munmap_interceptor(thr, pc, REAL(munmap), addr, sz); \ + } while (false) + #if !SANITIZER_APPLE #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ @@ -2588,6 +2545,8 @@ static __sanitizer_sighandler_ptr signal_impl(int sig, #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } +#define SIGNAL_INTERCEPTOR_ENTER() LazyInitialize(cur_thread_init()) + #include "sanitizer_common/sanitizer_signal_interceptors.inc" int sigaction_impl(int sig, const __sanitizer_sigaction *act, @@ -2608,7 +2567,7 @@ int sigaction_impl(int sig, const __sanitizer_sigaction *act, // Copy act into sigactions[sig]. // Can't use struct copy, because compiler can emit call to memcpy. // Can't use internal_memcpy, because it copies byte-by-byte, - // and signal handler reads the handler concurrently. It it can read + // and signal handler reads the handler concurrently. It can read // some bytes from old value and some bytes from new value. // Use volatile to prevent insertion of memcpy. sigactions[sig].handler = @@ -2893,16 +2852,7 @@ void InitializeInterceptors() { InitializeLibdispatchInterceptors(); #if !SANITIZER_APPLE - // We can not use TSAN_INTERCEPT to get setjmp addr, - // because it does &setjmp and setjmp is not present in some versions of libc. - using __interception::InterceptFunction; - InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); - InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); - InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, - 0); -#if !SANITIZER_NETBSD - InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); -#endif + InitializeSetjmpInterceptors(); #endif TSAN_INTERCEPT(longjmp_symname); @@ -3169,22 +3119,4 @@ SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( } } -void *__tsan_memcpy(void *dst, const void *src, uptr size) { - void *ctx; -#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); -#else - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -#endif -} - -void *__tsan_memset(void *dst, int c, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, c, size); -} - -void *__tsan_memmove(void *dst, const void *src, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -} -} +} // extern "C" diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h index d53c1e3935d..3731c90d459 100644 --- a/libsanitizer/tsan/tsan_interface.h +++ b/libsanitizer/tsan/tsan_interface.h @@ -419,6 +419,14 @@ void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a); SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a); +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a); diff --git a/libsanitizer/tsan/tsan_interface_ann.cpp b/libsanitizer/tsan/tsan_interface_ann.cpp index 6bd72e18d94..5154662034c 100644 --- a/libsanitizer/tsan/tsan_interface_ann.cpp +++ b/libsanitizer/tsan/tsan_interface_ann.cpp @@ -435,4 +435,26 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) { ThreadIgnoreBegin(thr, 0); ThreadIgnoreSyncBegin(thr, 0); } + +static void ReportMutexHeldWrongContext(ThreadState *thr, uptr pc) { + ThreadRegistryLock l(&ctx->thread_registry); + ScopedReport rep(ReportTypeMutexHeldWrongContext); + for (uptr i = 0; i < thr->mset.Size(); ++i) { + MutexSet::Desc desc = thr->mset.Get(i); + rep.AddMutex(desc.addr, desc.stack_id); + } + VarSizeStackTrace trace; + ObtainCurrentStack(thr, pc, &trace); + rep.AddStack(trace, true); + OutputReport(thr, rep); +} + +INTERFACE_ATTRIBUTE +void __tsan_check_no_mutexes_held() { + SCOPED_ANNOTATION(__tsan_check_no_mutexes_held); + if (thr->mset.Size() == 0) { + return; + } + ReportMutexHeldWrongContext(thr, pc); +} } // extern "C" diff --git a/libsanitizer/tsan/tsan_interface_atomic.cpp b/libsanitizer/tsan/tsan_interface_atomic.cpp index f794a2fcdd0..2b5a2c6ef79 100644 --- a/libsanitizer/tsan/tsan_interface_atomic.cpp +++ b/libsanitizer/tsan/tsan_interface_atomic.cpp @@ -895,6 +895,30 @@ void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { } SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAnd, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), + mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_and(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchAnd, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), + mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic32_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchOr, *(a32 *)(a + 16), *(a32 **)a, *(a32 *)(a + 8), + mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __tsan_go_atomic64_fetch_or(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { + ATOMIC_RET(FetchOr, *(a64 *)(a + 16), *(a64 **)a, *(a64 *)(a + 8), + mo_acq_rel); +} + +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) { ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel); } diff --git a/libsanitizer/tsan/tsan_malloc_mac.cpp b/libsanitizer/tsan/tsan_malloc_mac.cpp index ac844ae8a44..e973be963e5 100644 --- a/libsanitizer/tsan/tsan_malloc_mac.cpp +++ b/libsanitizer/tsan/tsan_malloc_mac.cpp @@ -17,6 +17,7 @@ #include "sanitizer_common/sanitizer_errno.h" #include "tsan_interceptors.h" #include "tsan_stack_trace.h" +#include "tsan_mman.h" using namespace __tsan; #define COMMON_MALLOC_ZONE_NAME "tsan" @@ -29,16 +30,30 @@ using namespace __tsan; user_memalign(cur_thread(), StackTrace::GetCurrentPc(), alignment, size) #define COMMON_MALLOC_MALLOC(size) \ if (in_symbolizer()) return InternalAlloc(size); \ - SCOPED_INTERCEPTOR_RAW(malloc, size); \ - void *p = user_alloc(thr, pc, size) + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(malloc, size); \ + p = user_alloc(thr, pc, size); \ + } \ + invoke_malloc_hook(p, size) #define COMMON_MALLOC_REALLOC(ptr, size) \ if (in_symbolizer()) return InternalRealloc(ptr, size); \ - SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ - void *p = user_realloc(thr, pc, ptr, size) + if (ptr) \ + invoke_free_hook(ptr); \ + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(realloc, ptr, size); \ + p = user_realloc(thr, pc, ptr, size); \ + } \ + invoke_malloc_hook(p, size) #define COMMON_MALLOC_CALLOC(count, size) \ if (in_symbolizer()) return InternalCalloc(count, size); \ - SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ - void *p = user_calloc(thr, pc, size, count) + void *p = 0; \ + { \ + SCOPED_INTERCEPTOR_RAW(calloc, size, count); \ + p = user_calloc(thr, pc, size, count); \ + } \ + invoke_malloc_hook(p, size * count) #define COMMON_MALLOC_POSIX_MEMALIGN(memptr, alignment, size) \ if (in_symbolizer()) { \ void *p = InternalAlloc(size, nullptr, alignment); \ @@ -55,6 +70,7 @@ using namespace __tsan; void *p = user_valloc(thr, pc, size) #define COMMON_MALLOC_FREE(ptr) \ if (in_symbolizer()) return InternalFree(ptr); \ + invoke_free_hook(ptr); \ SCOPED_INTERCEPTOR_RAW(free, ptr); \ user_free(thr, pc, ptr) #define COMMON_MALLOC_SIZE(ptr) uptr size = user_alloc_usable_size(ptr); diff --git a/libsanitizer/tsan/tsan_mman.cpp b/libsanitizer/tsan/tsan_mman.cpp index e5271cf5697..6f118e0979e 100644 --- a/libsanitizer/tsan/tsan_mman.cpp +++ b/libsanitizer/tsan/tsan_mman.cpp @@ -25,6 +25,8 @@ namespace __tsan { struct MapUnmapCallback { void OnMap(uptr p, uptr size) const { } + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const {}; void OnUnmap(uptr p, uptr size) const { // We are about to unmap a chunk of user memory. // Mark the corresponding shadow memory as not needed. @@ -377,6 +379,17 @@ uptr user_alloc_usable_size(const void *p) { return b->siz; } +uptr user_alloc_usable_size_fast(const void *p) { + MBlock *b = ctx->metamap.GetBlock((uptr)p); + // Static objects may have malloc'd before tsan completes + // initialization, and may believe returned ptrs to be valid. + if (!b) + return 0; // Not a valid pointer. + if (b->siz == 0) + return 1; // Zero-sized allocations are actually 1 byte. + return b->siz; +} + void invoke_malloc_hook(void *ptr, uptr size) { ThreadState *thr = cur_thread(); if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) @@ -452,6 +465,17 @@ uptr __sanitizer_get_allocated_size(const void *p) { return user_alloc_usable_size(p); } +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = user_alloc_usable_size_fast(p); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + +void __sanitizer_purge_allocator() { + allocator()->ForceReleaseToOS(); +} + void __tsan_on_thread_idle() { ThreadState *thr = cur_thread(); allocator()->SwallowCache(&thr->proc()->alloc_cache); diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index f0cdaf48eaa..d9ed3979b69 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -46,17 +46,16 @@ enum { /* C/C++ on linux/x86_64 and freebsd/x86_64 -0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) -0040 0000 0000 - 0100 0000 0000: - -0100 0000 0000 - 1000 0000 0000: shadow -1000 0000 0000 - 3000 0000 0000: - -3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects) -3400 0000 0000 - 5500 0000 0000: - -5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels -5680 0000 0000 - 7d00 0000 0000: - -7b00 0000 0000 - 7c00 0000 0000: heap -7c00 0000 0000 - 7e80 0000 0000: - -7e80 0000 0000 - 8000 0000 0000: modules and main thread stack +0000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB) +0200 0000 0000 - 1000 0000 0000: - +1000 0000 0000 - 3000 0000 0000: shadow (32TB) +3000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB) +3800 0000 0000 - 5500 0000 0000: - +5500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels +5a00 0000 0000 - 7200 0000 0000: - +7200 0000 0000 - 7300 0000 0000: heap (1TB) +7300 0000 0000 - 7a00 0000 0000: - +7a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB) C/C++ on netbsd/amd64 can reuse the same mapping: * The address space starts from 0x1000 (option with 0x0) and ends with @@ -72,20 +71,20 @@ C/C++ on netbsd/amd64 can reuse the same mapping: */ struct Mapping48AddressSpace { static const uptr kMetaShadowBeg = 0x300000000000ull; - static const uptr kMetaShadowEnd = 0x340000000000ull; - static const uptr kShadowBeg = 0x010000000000ull; - static const uptr kShadowEnd = 0x100000000000ull; - static const uptr kHeapMemBeg = 0x7b0000000000ull; - static const uptr kHeapMemEnd = 0x7c0000000000ull; + static const uptr kMetaShadowEnd = 0x380000000000ull; + static const uptr kShadowBeg = 0x100000000000ull; + static const uptr kShadowEnd = 0x300000000000ull; + static const uptr kHeapMemBeg = 0x720000000000ull; + static const uptr kHeapMemEnd = 0x730000000000ull; static const uptr kLoAppMemBeg = 0x000000001000ull; - static const uptr kLoAppMemEnd = 0x008000000000ull; + static const uptr kLoAppMemEnd = 0x020000000000ull; static const uptr kMidAppMemBeg = 0x550000000000ull; - static const uptr kMidAppMemEnd = 0x568000000000ull; - static const uptr kHiAppMemBeg = 0x7e8000000000ull; + static const uptr kMidAppMemEnd = 0x5a0000000000ull; + static const uptr kHiAppMemBeg = 0x7a0000000000ull; static const uptr kHiAppMemEnd = 0x800000000000ull; - static const uptr kShadowMsk = 0x780000000000ull; - static const uptr kShadowXor = 0x040000000000ull; - static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kShadowMsk = 0x700000000000ull; + static const uptr kShadowXor = 0x000000000000ull; + static const uptr kShadowAdd = 0x100000000000ull; static const uptr kVdsoBeg = 0xf000000000000000ull; }; @@ -378,6 +377,71 @@ struct MappingPPC64_47 { }; /* +C/C++ on linux/riscv64 (39-bit VMA) +0000 0010 00 - 0200 0000 00: main binary ( 8 GB) +0200 0000 00 - 1000 0000 00: - +1000 0000 00 - 4000 0000 00: shadow memory (64 GB) +4000 0000 00 - 4800 0000 00: metainfo (16 GB) +4800 0000 00 - 5500 0000 00: - +5500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB) +5600 0000 00 - 7c00 0000 00: - +7d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB) + +mmap by default allocates from top downwards +VDSO sits below loader and above dynamic libraries, within HiApp region. +Heap starts after program region whose position depends on pie or non-pie. +Disable tracking them since their locations are not fixed. +*/ +struct MappingRiscv64_39 { + static const uptr kLoAppMemBeg = 0x0000001000ull; + static const uptr kLoAppMemEnd = 0x0200000000ull; + static const uptr kShadowBeg = 0x1000000000ull; + static const uptr kShadowEnd = 0x2000000000ull; + static const uptr kMetaShadowBeg = 0x2000000000ull; + static const uptr kMetaShadowEnd = 0x2400000000ull; + static const uptr kMidAppMemBeg = 0x2aaaaaa000ull; + static const uptr kMidAppMemEnd = 0x2c00000000ull; + static const uptr kHeapMemBeg = 0x2c00000000ull; + static const uptr kHeapMemEnd = 0x2c00000000ull; + static const uptr kHiAppMemBeg = 0x3c00000000ull; + static const uptr kHiAppMemEnd = 0x3fffffffffull; + static const uptr kShadowMsk = 0x3800000000ull; + static const uptr kShadowXor = 0x0800000000ull; + static const uptr kShadowAdd = 0x0000000000ull; + static const uptr kVdsoBeg = 0x4000000000ull; +}; + +/* +C/C++ on linux/riscv64 (48-bit VMA) +0000 0000 1000 - 0500 0000 0000: main binary ( 5 TB) +0500 0000 0000 - 2000 0000 0000: - +2000 0000 0000 - 4000 0000 0000: shadow memory (32 TB) +4000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB) +4800 0000 0000 - 5555 5555 5000: - +5555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB) +5a00 0000 0000 - 7a00 0000 0000: - +7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 5 TB) +*/ +struct MappingRiscv64_48 { + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x050000000000ull; + static const uptr kShadowBeg = 0x200000000000ull; + static const uptr kShadowEnd = 0x400000000000ull; + static const uptr kMetaShadowBeg = 0x400000000000ull; + static const uptr kMetaShadowEnd = 0x480000000000ull; + static const uptr kMidAppMemBeg = 0x555555555000ull; + static const uptr kMidAppMemEnd = 0x5a0000000000ull; + static const uptr kHeapMemBeg = 0x5a0000000000ull; + static const uptr kHeapMemEnd = 0x5a0000000000ull; + static const uptr kHiAppMemBeg = 0x7a0000000000ull; + static const uptr kHiAppMemEnd = 0x7fffffffffffull; + static const uptr kShadowMsk = 0x700000000000ull; + static const uptr kShadowXor = 0x100000000000ull; + static const uptr kShadowAdd = 0x000000000000ull; + static const uptr kVdsoBeg = 0x800000000000ull; +}; + +/* C/C++ on linux/s390x While the kernel provides a 64-bit address space, we have to restrict ourselves to 48 bits due to how e.g. SyncVar::GetId() works. @@ -665,6 +729,13 @@ ALWAYS_INLINE auto SelectMapping(Arg arg) { } # elif defined(__mips64) return Func::template Apply<MappingMips64_40>(arg); +# elif SANITIZER_RISCV64 + switch (vmaSize) { + case 39: + return Func::template Apply<MappingRiscv64_39>(arg); + case 48: + return Func::template Apply<MappingRiscv64_48>(arg); + } # elif defined(__s390x__) return Func::template Apply<MappingS390x>(arg); # else @@ -686,6 +757,8 @@ void ForEachMapping() { Func::template Apply<MappingPPC64_44>(); Func::template Apply<MappingPPC64_46>(); Func::template Apply<MappingPPC64_47>(); + Func::template Apply<MappingRiscv64_39>(); + Func::template Apply<MappingRiscv64_48>(); Func::template Apply<MappingS390x>(); Func::template Apply<MappingGo48>(); Func::template Apply<MappingGoWindows>(); @@ -894,7 +967,7 @@ struct RestoreAddrImpl { Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd, Mapping::kHeapMemBeg, Mapping::kHeapMemEnd, }; - const uptr indicator = 0x0e0000000000ull; + const uptr indicator = 0x0f0000000000ull; const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator); for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) { uptr beg = ranges[i]; diff --git a/libsanitizer/tsan/tsan_platform_linux.cpp b/libsanitizer/tsan/tsan_platform_linux.cpp index 384a443c16b..369509ed0a6 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cpp +++ b/libsanitizer/tsan/tsan_platform_linux.cpp @@ -152,7 +152,7 @@ void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns) { #if !SANITIZER_GO // Mark shadow for .rodata sections with the special Shadow::kRodata marker. // Accesses to .rodata can't race, so this saves time, memory and trace space. -static void MapRodata() { +static NOINLINE void MapRodata(char* buffer, uptr size) { // First create temp file. const char *tmpdir = GetEnv("TMPDIR"); if (tmpdir == 0) @@ -163,13 +163,12 @@ static void MapRodata() { #endif if (tmpdir == 0) return; - char name[256]; - internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", + internal_snprintf(buffer, size, "%s/tsan.rodata.%d", tmpdir, (int)internal_getpid()); - uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); + uptr openrv = internal_open(buffer, O_RDWR | O_CREAT | O_EXCL, 0600); if (internal_iserror(openrv)) return; - internal_unlink(name); // Unlink it now, so that we can reuse the buffer. + internal_unlink(buffer); // Unlink it now, so that we can reuse the buffer. fd_t fd = openrv; // Fill the file with Shadow::kRodata. const uptr kMarkerSize = 512 * 1024 / sizeof(RawShadow); @@ -188,8 +187,8 @@ static void MapRodata() { } // Map the file into shadow of .rodata sections. MemoryMappingLayout proc_maps(/*cache_enabled*/true); - // Reusing the buffer 'name'. - MemoryMappedSegment segment(name, ARRAY_SIZE(name)); + // Reusing the buffer 'buffer'. + MemoryMappedSegment segment(buffer, size); while (proc_maps.Next(&segment)) { if (segment.filename[0] != 0 && segment.filename[0] != '[' && segment.IsReadable() && segment.IsExecutable() && @@ -209,7 +208,8 @@ static void MapRodata() { } void InitializeShadowMemoryPlatform() { - MapRodata(); + char buffer[256]; // Keep in a different frame. + MapRodata(buffer, sizeof(buffer)); } #endif // #if !SANITIZER_GO @@ -267,7 +267,17 @@ void InitializePlatformEarly() { Die(); } # endif -#endif +# elif SANITIZER_RISCV64 + // the bottom half of vma is allocated for userspace + vmaSize = vmaSize + 1; +# if !SANITIZER_GO + if (vmaSize != 39 && vmaSize != 48) { + Printf("FATAL: ThreadSanitizer: unsupported VMA range\n"); + Printf("FATAL: Found %zd - Supported 39 and 48\n", vmaSize); + Die(); + } +# endif +# endif } void InitializePlatform() { @@ -399,13 +409,15 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) { return mangled_sp ^ xor_key; #elif defined(__mips__) return mangled_sp; -#elif defined(__s390x__) +# elif SANITIZER_RISCV64 + return mangled_sp; +# elif defined(__s390x__) // tcbhead_t.stack_guard uptr xor_key = ((uptr *)__builtin_thread_pointer())[5]; return mangled_sp ^ xor_key; -#else - #error "Unknown platform" -#endif +# else +# error "Unknown platform" +# endif } #if SANITIZER_NETBSD @@ -429,11 +441,13 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) { # define LONG_JMP_SP_ENV_SLOT 1 # elif defined(__mips64) # define LONG_JMP_SP_ENV_SLOT 1 -# elif defined(__s390x__) -# define LONG_JMP_SP_ENV_SLOT 9 -# else -# define LONG_JMP_SP_ENV_SLOT 6 -# endif +# elif SANITIZER_RISCV64 +# define LONG_JMP_SP_ENV_SLOT 13 +# elif defined(__s390x__) +# define LONG_JMP_SP_ENV_SLOT 9 +# else +# define LONG_JMP_SP_ENV_SLOT 6 +# endif #endif uptr ExtractLongJmpSp(uptr *env) { diff --git a/libsanitizer/tsan/tsan_report.cpp b/libsanitizer/tsan/tsan_report.cpp index 9b03adc16b9..35cb6710a54 100644 --- a/libsanitizer/tsan/tsan_report.cpp +++ b/libsanitizer/tsan/tsan_report.cpp @@ -93,17 +93,13 @@ static const char *ReportTypeString(ReportType typ, uptr tag) { return "signal handler spoils errno"; case ReportTypeDeadlock: return "lock-order-inversion (potential deadlock)"; - // No default case so compiler warns us if we miss one + case ReportTypeMutexHeldWrongContext: + return "mutex held in the wrong context"; + // No default case so compiler warns us if we miss one } UNREACHABLE("missing case"); } -#if SANITIZER_APPLE -static const char *const kInterposedFunctionPrefix = "wrap_"; -#else -static const char *const kInterposedFunctionPrefix = "__interceptor_"; -#endif - void PrintStack(const ReportStack *ent) { if (ent == 0 || ent->frames == 0) { Printf(" [failed to restore the stack]\n\n"); @@ -112,10 +108,10 @@ void PrintStack(const ReportStack *ent) { SymbolizedStack *frame = ent->frames; for (int i = 0; frame && frame->info.address; frame = frame->next, i++) { InternalScopedString res; - RenderFrame(&res, common_flags()->stack_trace_format, i, - frame->info.address, &frame->info, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix, kInterposedFunctionPrefix); + StackTracePrinter::GetOrInit()->RenderFrame( + &res, common_flags()->stack_trace_format, i, frame->info.address, + &frame->info, common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); Printf("%s\n", res.data()); } Printf("\n"); @@ -284,6 +280,7 @@ static bool FrameIsInternal(const SymbolizedStack *frame) { const char *module = frame->info.module; if (file != 0 && (internal_strstr(file, "tsan_interceptors_posix.cpp") || + internal_strstr(file, "tsan_interceptors_memintrinsics.cpp") || internal_strstr(file, "sanitizer_common_interceptors.inc") || internal_strstr(file, "tsan_interface_"))) return true; diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h index 3c88864af14..bfe470797f8 100644 --- a/libsanitizer/tsan/tsan_report.h +++ b/libsanitizer/tsan/tsan_report.h @@ -34,7 +34,8 @@ enum ReportType { ReportTypeMutexBadReadUnlock, ReportTypeSignalUnsafe, ReportTypeErrnoInSignal, - ReportTypeDeadlock + ReportTypeDeadlock, + ReportTypeMutexHeldWrongContext }; struct ReportStack { diff --git a/libsanitizer/tsan/tsan_rtl.cpp b/libsanitizer/tsan/tsan_rtl.cpp index 6b1ec1d04fd..fd9441dfcb5 100644 --- a/libsanitizer/tsan/tsan_rtl.cpp +++ b/libsanitizer/tsan/tsan_rtl.cpp @@ -446,7 +446,7 @@ static bool InitializeMemoryProfiler() { ctx->memprof_fd = 2; } else { InternalScopedString filename; - filename.append("%s.%d", fname, (int)internal_getpid()); + filename.AppendF("%s.%d", fname, (int)internal_getpid()); ctx->memprof_fd = OpenFile(filename.data(), WrOnly); if (ctx->memprof_fd == kInvalidFd) { Printf("ThreadSanitizer: failed to open memory profile file '%s'\n", diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index a5606dbc7f8..de4ea0bb5f4 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -56,8 +56,8 @@ namespace __tsan { #if !SANITIZER_GO struct MapUnmapCallback; -#if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \ - defined(__powerpc__) +# if defined(__mips64) || defined(__aarch64__) || defined(__loongarch__) || \ + defined(__powerpc__) || SANITIZER_RISCV64 struct AP32 { static const uptr kSpaceBeg = 0; diff --git a/libsanitizer/tsan/tsan_rtl_ppc64.S b/libsanitizer/tsan/tsan_rtl_ppc64.S index 9e533a71a9c..8285e21aa1e 100644 --- a/libsanitizer/tsan/tsan_rtl_ppc64.S +++ b/libsanitizer/tsan/tsan_rtl_ppc64.S @@ -1,6 +1,5 @@ #include "tsan_ppc_regs.h" - .machine altivec .section .text .hidden __tsan_setjmp .globl _setjmp diff --git a/libsanitizer/tsan/tsan_rtl_riscv64.S b/libsanitizer/tsan/tsan_rtl_riscv64.S new file mode 100644 index 00000000000..8e6b9b9432e --- /dev/null +++ b/libsanitizer/tsan/tsan_rtl_riscv64.S @@ -0,0 +1,203 @@ +#include "sanitizer_common/sanitizer_asm.h" + +.section .text + +.comm _ZN14__interception11real_setjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(setjmp)) +ASM_SYMBOL_INTERCEPTOR(setjmp): + CFI_STARTPROC + + // Save frame pointer and return address register + addi sp, sp, -32 + sd ra, 24(sp) + sd s0, 16(sp) + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (1, -8) + CFI_OFFSET (8, -16) + + // Adjust the SP for previous frame + addi s0, sp, 32 + CFI_DEF_CFA_REGISTER (8) + + // Save env parameter + sd a0, 8(sp) + CFI_OFFSET (10, -24) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + addi a0, s0, 0 + + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ld a0, 8(sp) + CFI_RESTORE (10) + + // Restore frame/link register + ld s0, 16(sp) + ld ra, 24(sp) + addi sp, sp, 32 + CFI_RESTORE (8) + CFI_RESTORE (1) + CFI_DEF_CFA (2, 0) + + // tail jump to libc setjmp + la t1, _ZN14__interception11real_setjmpE + ld t1, 0(t1) + jr t1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(setjmp)) + +.comm _ZN14__interception12real__setjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(_setjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(_setjmp)) +ASM_SYMBOL_INTERCEPTOR(_setjmp): + CFI_STARTPROC + + // Save frame pointer and return address register + addi sp, sp, -32 + sd ra, 24(sp) + sd s0, 16(sp) + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (1, -8) + CFI_OFFSET (8, -16) + + // Adjust the SP for previous frame + addi s0, sp, 32 + CFI_DEF_CFA_REGISTER (8) + + // Save env parameter + sd a0, 8(sp) + CFI_OFFSET (10, -24) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + addi a0, s0, 0 + + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ld a0, 8(sp) + CFI_RESTORE (10) + + // Restore frame/link register + ld s0, 16(sp) + ld ra, 24(sp) + addi sp, sp, 32 + CFI_RESTORE (8) + CFI_RESTORE (1) + CFI_DEF_CFA (2, 0) + + // tail jump to libc setjmp + la t1, _ZN14__interception12real__setjmpE + ld t1, 0(t1) + jr t1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(_setjmp)) + +.comm _ZN14__interception14real_sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(sigsetjmp): + CFI_STARTPROC + + // Save frame pointer and return address register + addi sp, sp, -32 + sd ra, 24(sp) + sd s0, 16(sp) + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (1, -8) + CFI_OFFSET (8, -16) + + // Adjust the SP for previous frame + addi s0, sp, 32 + CFI_DEF_CFA_REGISTER (8) + + // Save env parameter + sd a0, 8(sp) + sd a1, 0(sp) + CFI_OFFSET (10, -24) + CFI_OFFSET (11, -32) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + addi a0, s0, 0 + + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ld a0, 8(sp) + ld a1, 0(sp) + CFI_RESTORE (10) + CFI_RESTORE (11) + + // Restore frame/link register + ld s0, 16(sp) + ld ra, 24(sp) + addi sp, sp, 32 + CFI_RESTORE (8) + CFI_RESTORE (1) + CFI_DEF_CFA (2, 0) + + // tail jump to libc setjmp + la t1, _ZN14__interception14real_sigsetjmpE + ld t1, 0(t1) + jr t1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(sigsetjmp)) + +.comm _ZN14__interception16real___sigsetjmpE,8,8 +.globl ASM_SYMBOL_INTERCEPTOR(__sigsetjmp) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) +ASM_SYMBOL_INTERCEPTOR(__sigsetjmp): + CFI_STARTPROC + + // Save frame pointer and return address register + addi sp, sp, -32 + sd ra, 24(sp) + sd s0, 16(sp) + CFI_DEF_CFA_OFFSET (32) + CFI_OFFSET (1, -8) + CFI_OFFSET (8, -16) + + // Adjust the SP for previous frame + addi s0, sp, 32 + CFI_DEF_CFA_REGISTER (8) + + // Save env parameter + sd a0, 8(sp) + sd a1, 0(sp) + CFI_OFFSET (10, -24) + CFI_OFFSET (11, -32) + + // Obtain SP, first argument to `void __tsan_setjmp(uptr sp)` + addi a0, s0, 0 + + // call tsan interceptor + call ASM_SYMBOL(__tsan_setjmp) + + // Restore env parameter + ld a0, 8(sp) + ld a1, 0(sp) + CFI_RESTORE (10) + CFI_RESTORE (11) + + // Restore frame/link register + ld s0, 16(sp) + ld ra, 24(sp) + addi sp, sp, 32 + CFI_RESTORE (8) + CFI_RESTORE (1) + CFI_DEF_CFA (2, 0) + + // tail jump to libc setjmp + la t1, _ZN14__interception16real___sigsetjmpE + ld t1, 0(t1) + jr t1 + + CFI_ENDPROC +ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp)) diff --git a/libsanitizer/tsan/tsan_suppressions.cpp b/libsanitizer/tsan/tsan_suppressions.cpp index 9cdfa32a934..70642124990 100644 --- a/libsanitizer/tsan/tsan_suppressions.cpp +++ b/libsanitizer/tsan/tsan_suppressions.cpp @@ -81,6 +81,7 @@ static const char *conv(ReportType typ) { case ReportTypeMutexBadUnlock: case ReportTypeMutexBadReadLock: case ReportTypeMutexBadReadUnlock: + case ReportTypeMutexHeldWrongContext: return kSuppressionMutex; case ReportTypeSignalUnsafe: case ReportTypeErrnoInSignal: diff --git a/libsanitizer/ubsan/ubsan_diag.cpp b/libsanitizer/ubsan/ubsan_diag.cpp index dd99613abbe..aac27041531 100644 --- a/libsanitizer/ubsan/ubsan_diag.cpp +++ b/libsanitizer/ubsan/ubsan_diag.cpp @@ -134,9 +134,9 @@ Diag &Diag::operator<<(const Value &V) { /// Hexadecimal printing for numbers too large for Printf to handle directly. static void RenderHex(InternalScopedString *Buffer, UIntMax Val) { #if HAVE_INT128_T - Buffer->append("0x%08x%08x%08x%08x", (unsigned int)(Val >> 96), - (unsigned int)(Val >> 64), (unsigned int)(Val >> 32), - (unsigned int)(Val)); + Buffer->AppendF("0x%08x%08x%08x%08x", (unsigned int)(Val >> 96), + (unsigned int)(Val >> 64), (unsigned int)(Val >> 32), + (unsigned int)(Val)); #else UNREACHABLE("long long smaller than 64 bits?"); #endif @@ -147,31 +147,34 @@ static void RenderLocation(InternalScopedString *Buffer, Location Loc) { case Location::LK_Source: { SourceLocation SLoc = Loc.getSourceLocation(); if (SLoc.isInvalid()) - Buffer->append("<unknown>"); + Buffer->AppendF("<unknown>"); else - RenderSourceLocation(Buffer, SLoc.getFilename(), SLoc.getLine(), - SLoc.getColumn(), common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderSourceLocation( + Buffer, SLoc.getFilename(), SLoc.getLine(), SLoc.getColumn(), + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); return; } case Location::LK_Memory: - Buffer->append("%p", reinterpret_cast<void *>(Loc.getMemoryLocation())); + Buffer->AppendF("%p", reinterpret_cast<void *>(Loc.getMemoryLocation())); return; case Location::LK_Symbolized: { const AddressInfo &Info = Loc.getSymbolizedStack()->info; if (Info.file) - RenderSourceLocation(Buffer, Info.file, Info.line, Info.column, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderSourceLocation( + Buffer, Info.file, Info.line, Info.column, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); else if (Info.module) - RenderModuleLocation(Buffer, Info.module, Info.module_offset, - Info.module_arch, common_flags()->strip_path_prefix); + StackTracePrinter::GetOrInit()->RenderModuleLocation( + Buffer, Info.module, Info.module_offset, Info.module_arch, + common_flags()->strip_path_prefix); else - Buffer->append("%p", reinterpret_cast<void *>(Info.address)); + Buffer->AppendF("%p", reinterpret_cast<void *>(Info.address)); return; } case Location::LK_Null: - Buffer->append("<unknown>"); + Buffer->AppendF("<unknown>"); return; } } @@ -180,32 +183,32 @@ static void RenderText(InternalScopedString *Buffer, const char *Message, const Diag::Arg *Args) { for (const char *Msg = Message; *Msg; ++Msg) { if (*Msg != '%') { - Buffer->append("%c", *Msg); + Buffer->AppendF("%c", *Msg); continue; } const Diag::Arg &A = Args[*++Msg - '0']; switch (A.Kind) { case Diag::AK_String: - Buffer->append("%s", A.String); + Buffer->AppendF("%s", A.String); break; case Diag::AK_TypeName: { if (SANITIZER_WINDOWS) // The Windows implementation demangles names early. - Buffer->append("'%s'", A.String); + Buffer->AppendF("'%s'", A.String); else - Buffer->append("'%s'", Symbolizer::GetOrInit()->Demangle(A.String)); + Buffer->AppendF("'%s'", Symbolizer::GetOrInit()->Demangle(A.String)); break; } case Diag::AK_SInt: // 'long long' is guaranteed to be at least 64 bits wide. if (A.SInt >= INT64_MIN && A.SInt <= INT64_MAX) - Buffer->append("%lld", (long long)A.SInt); + Buffer->AppendF("%lld", (long long)A.SInt); else RenderHex(Buffer, A.SInt); break; case Diag::AK_UInt: if (A.UInt <= UINT64_MAX) - Buffer->append("%llu", (unsigned long long)A.UInt); + Buffer->AppendF("%llu", (unsigned long long)A.UInt); else RenderHex(Buffer, A.UInt); break; @@ -223,11 +226,11 @@ static void RenderText(InternalScopedString *Buffer, const char *Message, #else snprintf(FloatBuffer, sizeof(FloatBuffer), "%Lg", (long double)A.Float); #endif - Buffer->append("%s", FloatBuffer); + Buffer->Append(FloatBuffer); break; } case Diag::AK_Pointer: - Buffer->append("%p", A.Pointer); + Buffer->AppendF("%p", A.Pointer); break; } } @@ -284,12 +287,12 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, InternalScopedString Buffer; for (uptr P = Min; P != Max; ++P) { unsigned char C = *reinterpret_cast<const unsigned char*>(P); - Buffer.append("%s%02x", (P % 8 == 0) ? " " : " ", C); + Buffer.AppendF("%s%02x", (P % 8 == 0) ? " " : " ", C); } - Buffer.append("\n"); + Buffer.AppendF("\n"); // Emit highlights. - Buffer.append("%s", Decor.Highlight()); + Buffer.Append(Decor.Highlight()); Range *InRange = upperBound(Min, Ranges, NumRanges); for (uptr P = Min; P != Max; ++P) { char Pad = ' ', Byte = ' '; @@ -302,12 +305,12 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, if (InRange && InRange->getStart().getMemoryLocation() <= P) Byte = '~'; if (P % 8 == 0) - Buffer.append("%c", Pad); - Buffer.append("%c", Pad); - Buffer.append("%c", P == Loc ? '^' : Byte); - Buffer.append("%c", Byte); + Buffer.AppendF("%c", Pad); + Buffer.AppendF("%c", Pad); + Buffer.AppendF("%c", P == Loc ? '^' : Byte); + Buffer.AppendF("%c", Byte); } - Buffer.append("%s\n", Decor.Default()); + Buffer.AppendF("%s\n", Decor.Default()); // Go over the line again, and print names for the ranges. InRange = 0; @@ -322,9 +325,9 @@ static void PrintMemorySnippet(const Decorator &Decor, MemoryLocation Loc, if (InRange && InRange->getStart().getMemoryLocation() == P) { while (Spaces--) - Buffer.append(" "); + Buffer.AppendF(" "); RenderText(&Buffer, InRange->getText(), Args); - Buffer.append("\n"); + Buffer.AppendF("\n"); // FIXME: We only support naming one range for now! break; } @@ -358,24 +361,24 @@ Diag::~Diag() { Buffer.clear(); } - Buffer.append("%s", Decor.Bold()); + Buffer.Append(Decor.Bold()); RenderLocation(&Buffer, Loc); - Buffer.append(":"); + Buffer.AppendF(":"); switch (Level) { case DL_Error: - Buffer.append("%s runtime error: %s%s", Decor.Warning(), Decor.Default(), - Decor.Bold()); + Buffer.AppendF("%s runtime error: %s%s", Decor.Warning(), Decor.Default(), + Decor.Bold()); break; case DL_Note: - Buffer.append("%s note: %s", Decor.Note(), Decor.Default()); + Buffer.AppendF("%s note: %s", Decor.Note(), Decor.Default()); break; } RenderText(&Buffer, Message, Args); - Buffer.append("%s\n", Decor.Default()); + Buffer.AppendF("%s\n", Decor.Default()); Printf("%s", Buffer.data()); if (Loc.isMemoryLocation()) diff --git a/libsanitizer/ubsan/ubsan_flags.cpp b/libsanitizer/ubsan/ubsan_flags.cpp index 9a66bd37518..25cefd46ce2 100644 --- a/libsanitizer/ubsan/ubsan_flags.cpp +++ b/libsanitizer/ubsan/ubsan_flags.cpp @@ -50,7 +50,6 @@ void InitializeFlags() { { CommonFlags cf; cf.CopyFrom(*common_flags()); - cf.print_summary = false; cf.external_symbolizer_path = GetFlag("UBSAN_SYMBOLIZER_PATH"); OverrideCommonFlags(cf); } diff --git a/libsanitizer/ubsan/ubsan_handlers.cpp b/libsanitizer/ubsan/ubsan_handlers.cpp index 970075e69a6..0f16507d5d8 100644 --- a/libsanitizer/ubsan/ubsan_handlers.cpp +++ b/libsanitizer/ubsan/ubsan_handlers.cpp @@ -894,21 +894,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, } // namespace __ubsan -void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(false); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); -} - -void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(true); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); - Die(); -} - void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, ValueHandle Value, uptr ValidVtable) { @@ -930,4 +915,39 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, Die(); } +static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, + ValueHandle Function, + ReportOptions Opts) { + SourceLocation CallLoc = Data->Loc.acquire(); + ErrorType ET = ErrorType::FunctionTypeMismatch; + if (ignoreReport(CallLoc, Opts, ET)) + return true; + + ScopedReport R(Opts, CallLoc, ET); + + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; + + Diag(CallLoc, DL_Error, ET, + "call to function %0 through pointer to incorrect function type %1") + << FName << Data->Type; + Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; + return true; +} + +void __ubsan::__ubsan_handle_function_type_mismatch( + FunctionTypeMismatchData *Data, ValueHandle Function) { + GET_REPORT_OPTIONS(false); + handleFunctionTypeMismatch(Data, Function, Opts); +} + +void __ubsan::__ubsan_handle_function_type_mismatch_abort( + FunctionTypeMismatchData *Data, ValueHandle Function) { + GET_REPORT_OPTIONS(true); + if (handleFunctionTypeMismatch(Data, Function, Opts)) + Die(); +} + #endif // CAN_SANITIZE_UB diff --git a/libsanitizer/ubsan/ubsan_handlers.h b/libsanitizer/ubsan/ubsan_handlers.h index 9f412353fc0..3bd5046de3d 100644 --- a/libsanitizer/ubsan/ubsan_handlers.h +++ b/libsanitizer/ubsan/ubsan_handlers.h @@ -215,20 +215,12 @@ enum CFITypeCheckKind : unsigned char { CFITCK_VMFCall, }; -struct CFIBadIcallData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - struct CFICheckFailData { CFITypeCheckKind CheckKind; SourceLocation Loc; const TypeDescriptor &Type; }; -/// \brief Handle control flow integrity failure for indirect function calls. -RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function) - /// \brief Handle control flow integrity failures. RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function, uptr VtableIsValid) @@ -239,6 +231,17 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type( CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable, ReportOptions Opts); +struct FunctionTypeMismatchData { + SourceLocation Loc; + const TypeDescriptor &Type; +}; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, + ValueHandle Val); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch_abort(FunctionTypeMismatchData *Data, + ValueHandle Val); } #endif // UBSAN_HANDLERS_H diff --git a/libsanitizer/ubsan/ubsan_handlers_cxx.cpp b/libsanitizer/ubsan/ubsan_handlers_cxx.cpp index 0317a3d1428..206a0bb485a 100644 --- a/libsanitizer/ubsan/ubsan_handlers_cxx.cpp +++ b/libsanitizer/ubsan/ubsan_handlers_cxx.cpp @@ -156,50 +156,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, Diag(Loc, DL_Note, ET, "check failed in %0, vtable located in %1") << SrcModule << DstModule; } - -static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, - ValueHandle Function, - ValueHandle calleeRTTI, - ValueHandle fnRTTI, ReportOptions Opts) { - if (checkTypeInfoEquality(reinterpret_cast<void *>(calleeRTTI), - reinterpret_cast<void *>(fnRTTI))) - return false; - - SourceLocation CallLoc = Data->Loc.acquire(); - ErrorType ET = ErrorType::FunctionTypeMismatch; - - if (ignoreReport(CallLoc, Opts, ET)) - return true; - - ScopedReport R(Opts, CallLoc, ET); - - SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); - const char *FName = FLoc.get()->info.function; - if (!FName) - FName = "(unknown)"; - - Diag(CallLoc, DL_Error, ET, - "call to function %0 through pointer to incorrect function type %1") - << FName << Data->Type; - Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; - return true; -} - -void __ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, - ValueHandle Function, - ValueHandle calleeRTTI, - ValueHandle fnRTTI) { - GET_REPORT_OPTIONS(false); - handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts); -} - -void __ubsan_handle_function_type_mismatch_v1_abort( - FunctionTypeMismatchData *Data, ValueHandle Function, - ValueHandle calleeRTTI, ValueHandle fnRTTI) { - GET_REPORT_OPTIONS(true); - if (handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts)) - Die(); -} } // namespace __ubsan #endif // CAN_SANITIZE_UB diff --git a/libsanitizer/ubsan/ubsan_handlers_cxx.h b/libsanitizer/ubsan/ubsan_handlers_cxx.h index fd534c2573f..71695cbdc09 100644 --- a/libsanitizer/ubsan/ubsan_handlers_cxx.h +++ b/libsanitizer/ubsan/ubsan_handlers_cxx.h @@ -33,22 +33,6 @@ void __ubsan_handle_dynamic_type_cache_miss( extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_dynamic_type_cache_miss_abort( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash); - -struct FunctionTypeMismatchData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - -extern "C" SANITIZER_INTERFACE_ATTRIBUTE void -__ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, - ValueHandle Val, - ValueHandle calleeRTTI, - ValueHandle fnRTTI); -extern "C" SANITIZER_INTERFACE_ATTRIBUTE void -__ubsan_handle_function_type_mismatch_v1_abort(FunctionTypeMismatchData *Data, - ValueHandle Val, - ValueHandle calleeRTTI, - ValueHandle fnRTTI); } #endif // UBSAN_HANDLERS_CXX_H diff --git a/libsanitizer/ubsan/ubsan_interface.inc b/libsanitizer/ubsan/ubsan_interface.inc index 94337d85017..cb27feb5d7e 100644 --- a/libsanitizer/ubsan/ubsan_interface.inc +++ b/libsanitizer/ubsan/ubsan_interface.inc @@ -21,8 +21,8 @@ INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss) INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1_abort) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion_abort) INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin) diff --git a/libsanitizer/ubsan/ubsan_monitor.cpp b/libsanitizer/ubsan/ubsan_monitor.cpp index 69dd986f9bd..caed9726d48 100644 --- a/libsanitizer/ubsan/ubsan_monitor.cpp +++ b/libsanitizer/ubsan/ubsan_monitor.cpp @@ -23,7 +23,8 @@ UndefinedBehaviorReport::UndefinedBehaviorReport(const char *IssueKind, RegisterUndefinedBehaviorReport(this); // Make a copy of the diagnostic. - Buffer.append("%s", Msg.data()); + if (Msg.length()) + Buffer.Append(Msg.data()); // Let the monitor know that a report is available. __ubsan_on_report(); diff --git a/libsanitizer/ubsan/ubsan_platform.h b/libsanitizer/ubsan/ubsan_platform.h index ad3e883f0f3..d2cc2e10bd2 100644 --- a/libsanitizer/ubsan/ubsan_platform.h +++ b/libsanitizer/ubsan/ubsan_platform.h @@ -12,7 +12,6 @@ #ifndef UBSAN_PLATFORM_H #define UBSAN_PLATFORM_H -#ifndef CAN_SANITIZE_UB // Other platforms should be easy to add, and probably work as-is. #if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \ defined(__NetBSD__) || defined(__DragonFly__) || \ @@ -22,6 +21,5 @@ #else # define CAN_SANITIZE_UB 0 #endif -#endif //CAN_SANITIZE_UB #endif diff --git a/libsanitizer/ubsan/ubsan_signals_standalone.cpp b/libsanitizer/ubsan/ubsan_signals_standalone.cpp index 2c91db8ca39..354f847fab7 100644 --- a/libsanitizer/ubsan/ubsan_signals_standalone.cpp +++ b/libsanitizer/ubsan/ubsan_signals_standalone.cpp @@ -34,7 +34,12 @@ void InitializeDeadlySignals() {} #else +namespace __ubsan { +void InitializeDeadlySignals(); +} // namespace __ubsan + #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define SIGNAL_INTERCEPTOR_ENTER() __ubsan::InitializeDeadlySignals() #include "sanitizer_common/sanitizer_signal_interceptors.inc" // TODO(yln): Temporary workaround. Will be removed. |