system/wine
system/wine
1
0
Fork 0
mirror of git://source.winehq.org/git/wine.git synced 2024-07-21 22:34:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
wine/dlls/ntdll/heap.c
Paul Gofman c77642ec52 ntdll: Match Windows used block filling. 
Test rewritten by Rémi Bernon.
2023-07-27 13:12:06 +09:00

2749 lines
97 KiB
C
Raw Blame History

/*
* Win32 heap functions
*
* Copyright 1996 Alexandre Julliard
* Copyright 1998 Ulrich Weigand
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
*/
#include <assert.h>
#include <stdlib.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#define RUNNING_ON_VALGRIND 0 /* FIXME */
#include "ntstatus.h"
#define WIN32_NO_STATUS
#include "windef.h"
#include "winnt.h"
#include "winternl.h"
#include "ntdll_misc.h"
#include "wine/list.h"
#include "wine/debug.h"
WINE_DEFAULT_DEBUG_CHANNEL(heap);
/* HeapCompatibilityInformation values */
#define HEAP_STD 0
#define HEAP_LAL 1
#define HEAP_LFH 2
/* undocumented RtlWalkHeap structure */
struct rtl_heap_entry
{
LPVOID lpData;
SIZE_T cbData; /* differs from PROCESS_HEAP_ENTRY */
BYTE cbOverhead;
BYTE iRegionIndex;
WORD wFlags; /* value differs from PROCESS_HEAP_ENTRY */
union {
struct {
HANDLE hMem;
DWORD dwReserved[3];
} Block;
struct {
DWORD dwCommittedSize;
DWORD dwUnCommittedSize;
LPVOID lpFirstBlock;
LPVOID lpLastBlock;
} Region;
};
};
/* rtl_heap_entry flags, names made up */
#define RTL_HEAP_ENTRY_BUSY 0x0001
#define RTL_HEAP_ENTRY_REGION 0x0002
#define RTL_HEAP_ENTRY_BLOCK 0x0010
#define RTL_HEAP_ENTRY_UNCOMMITTED 0x1000
#define RTL_HEAP_ENTRY_COMMITTED 0x4000
#define RTL_HEAP_ENTRY_LFH 0x8000
/* header for heap blocks */
#define REGION_ALIGN 0x10000
#define BLOCK_ALIGN (2 * sizeof(void *))
struct DECLSPEC_ALIGN(8) block
{
/* block size in multiple of BLOCK_ALIGN */
WORD block_size;
/* unused size (used block) / high size bits (free block) */
WORD tail_size;
/* offset to region base / first group block (LFH block) */
WORD base_offset;
BYTE block_type;
BYTE block_flags;
};
C_ASSERT( sizeof(struct block) == 8 );
/* block specific flags */
#define BLOCK_FLAG_FREE 0x01
#define BLOCK_FLAG_PREV_FREE 0x02
#define BLOCK_FLAG_FREE_LINK 0x03
#define BLOCK_FLAG_LARGE 0x04
#define BLOCK_FLAG_LFH 0x08 /* block is handled by the LFH frontend */
#define BLOCK_FLAG_USER_INFO 0x10 /* user flags up to 0xf0 */
#define BLOCK_FLAG_USER_MASK 0xf0
#define BLOCK_USER_FLAGS( heap_flags ) (((heap_flags) >> 4) & BLOCK_FLAG_USER_MASK)
#define HEAP_USER_FLAGS( block_flags ) (((block_flags) & BLOCK_FLAG_USER_MASK) << 4)
/* entry to link free blocks in free lists */
struct DECLSPEC_ALIGN(BLOCK_ALIGN) entry
{
struct block block;
struct list entry;
};
C_ASSERT( sizeof(struct entry) == 2 * BLOCK_ALIGN );
typedef struct
{
SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)];
struct list entry; /* entry in heap large blocks list */
SIZE_T data_size; /* size of user data */
SIZE_T block_size; /* total size of virtual memory block */
void *user_value;
struct block block;
} ARENA_LARGE;
/* block must be last and aligned */
C_ASSERT( sizeof(ARENA_LARGE) == offsetof(ARENA_LARGE, block) + sizeof(struct block) );
C_ASSERT( sizeof(ARENA_LARGE) == 4 * BLOCK_ALIGN );
#define BLOCK_TYPE_USED 'u'
#define BLOCK_TYPE_DEAD 'D'
#define BLOCK_TYPE_FREE 'F'
#define BLOCK_TYPE_LARGE 'L'
#define BLOCK_FILL_USED 0xbaadf00d
#define BLOCK_FILL_TAIL 0xab
#define BLOCK_FILL_FREE 0xfeeefeee
#define ROUND_ADDR(addr, mask) ((void *)((UINT_PTR)(addr) & ~(UINT_PTR)(mask)))
#define ROUND_SIZE(size, mask) ((((SIZE_T)(size) + (mask)) & ~(SIZE_T)(mask)))
#define FIELD_BITS(type, field) (sizeof(((type *)0)->field) * 8)
#define FIELD_MAX(type, field) (((SIZE_T)1 << FIELD_BITS(type, field)) - 1)
#define HEAP_MIN_BLOCK_SIZE ROUND_SIZE(sizeof(struct entry) + BLOCK_ALIGN, BLOCK_ALIGN - 1)
C_ASSERT( sizeof(struct block) <= HEAP_MIN_BLOCK_SIZE );
C_ASSERT( sizeof(struct entry) <= HEAP_MIN_BLOCK_SIZE );
/* used block size is coded into block_size */
#define HEAP_MAX_USED_BLOCK_SIZE (FIELD_MAX( struct block, block_size ) * BLOCK_ALIGN)
/* free block size is coded into block_size + tail_size */
#define HEAP_MAX_FREE_BLOCK_SIZE (HEAP_MAX_USED_BLOCK_SIZE + (FIELD_MAX( struct block, tail_size ) << 16) * BLOCK_ALIGN)
/* subheap distance is coded into base_offset */
#define HEAP_MAX_BLOCK_REGION_SIZE (FIELD_MAX( struct block, base_offset ) * REGION_ALIGN)
C_ASSERT( HEAP_MAX_USED_BLOCK_SIZE == 512 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
C_ASSERT( HEAP_MAX_FREE_BLOCK_SIZE >= 128 * 1024 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
C_ASSERT( HEAP_MAX_BLOCK_REGION_SIZE >= 128 * 1024 * 1024 * (sizeof(void *) / 4) - BLOCK_ALIGN );
C_ASSERT( HEAP_MAX_FREE_BLOCK_SIZE >= HEAP_MAX_BLOCK_REGION_SIZE );
/* minimum size to start allocating large blocks */
#define HEAP_MIN_LARGE_BLOCK_SIZE (HEAP_MAX_USED_BLOCK_SIZE - 0x1000)
#define FREE_LIST_LINEAR_BITS 2
#define FREE_LIST_LINEAR_MASK ((1 << FREE_LIST_LINEAR_BITS) - 1)
#define FREE_LIST_COUNT ((FIELD_BITS( struct block, block_size ) - FREE_LIST_LINEAR_BITS + 1) * (1 << FREE_LIST_LINEAR_BITS) + 1)
/* for reference, update this when changing parameters */
C_ASSERT( FREE_LIST_COUNT == 0x3d );
typedef struct DECLSPEC_ALIGN(BLOCK_ALIGN) tagSUBHEAP
{
SIZE_T __pad[sizeof(SIZE_T) / sizeof(DWORD)];
SIZE_T block_size;
SIZE_T data_size;
struct list entry;
void *user_value;
struct block block;
} SUBHEAP;
/* block must be last and aligned */
C_ASSERT( sizeof(SUBHEAP) == offsetof(SUBHEAP, block) + sizeof(struct block) );
C_ASSERT( sizeof(SUBHEAP) == 4 * BLOCK_ALIGN );
/* LFH block size bins */
#define BIN_SIZE_MIN_0 0
#define BIN_SIZE_MIN_1 0x100
#define BIN_SIZE_MIN_2 0x200
#define BIN_SIZE_MIN_3 0x400
#define BIN_SIZE_MIN_4 0x800
#define BIN_SIZE_MIN_5 0x1000
#define BIN_SIZE_MIN_6 0x2000
#define BIN_SIZE_MIN_7 0x4000
#define BIN_SIZE_MAX 0x8000
#define BIN_SIZE_STEP_0 (16)
#define BIN_SIZE_STEP_1 (BIN_SIZE_MIN_1 >> 4)
#define BIN_SIZE_STEP_2 (BIN_SIZE_MIN_2 >> 4)
#define BIN_SIZE_STEP_3 (BIN_SIZE_MIN_3 >> 4)
#define BIN_SIZE_STEP_4 (BIN_SIZE_MIN_4 >> 4)
#define BIN_SIZE_STEP_5 (BIN_SIZE_MIN_5 >> 4)
#define BIN_SIZE_STEP_6 (BIN_SIZE_MIN_6 >> 4)
#define BIN_SIZE_STEP_7 (BIN_SIZE_MIN_7 >> 4)
#define BLOCK_BIN_SIZE_N( n, bin ) (BIN_SIZE_MIN_##n + (bin + 1) * BIN_SIZE_STEP_##n)
#define BLOCK_SIZE_BIN_N( n, size ) (((size) - 1 - BIN_SIZE_MIN_##n) / BIN_SIZE_STEP_##n)
#define BLOCK_BIN_SIZE( bin ) ((bin) >= 0x80 ? ~(SIZE_T)0 : \
(bin) >= 0x70 ? BLOCK_BIN_SIZE_N( 7, bin - 0x70 ) : \
(bin) >= 0x60 ? BLOCK_BIN_SIZE_N( 6, bin - 0x60 ) : \
(bin) >= 0x50 ? BLOCK_BIN_SIZE_N( 5, bin - 0x50 ) : \
(bin) >= 0x40 ? BLOCK_BIN_SIZE_N( 4, bin - 0x40 ) : \
(bin) >= 0x30 ? BLOCK_BIN_SIZE_N( 3, bin - 0x30 ) : \
(bin) >= 0x20 ? BLOCK_BIN_SIZE_N( 2, bin - 0x20 ) : \
(bin) >= 0x10 ? BLOCK_BIN_SIZE_N( 1, bin - 0x10 ) : \
BLOCK_BIN_SIZE_N( 0, bin ))
#define BLOCK_SIZE_BIN( size ) ((size) > BIN_SIZE_MAX ? 0x80 : \
(size) > BIN_SIZE_MIN_7 ? 0x70 + BLOCK_SIZE_BIN_N( 7, size ) : \
(size) > BIN_SIZE_MIN_6 ? 0x60 + BLOCK_SIZE_BIN_N( 6, size ) : \
(size) > BIN_SIZE_MIN_5 ? 0x50 + BLOCK_SIZE_BIN_N( 5, size ) : \
(size) > BIN_SIZE_MIN_4 ? 0x40 + BLOCK_SIZE_BIN_N( 4, size ) : \
(size) > BIN_SIZE_MIN_3 ? 0x30 + BLOCK_SIZE_BIN_N( 3, size ) : \
(size) > BIN_SIZE_MIN_2 ? 0x20 + BLOCK_SIZE_BIN_N( 2, size ) : \
(size) > BIN_SIZE_MIN_1 ? 0x10 + BLOCK_SIZE_BIN_N( 1, size ) : \
(size) <= BIN_SIZE_MIN_0 ? 0 : BLOCK_SIZE_BIN_N( 0, size ))
#define BLOCK_SIZE_BIN_COUNT (BLOCK_SIZE_BIN( BIN_SIZE_MAX + 1 ) + 1)
/* macros sanity checks */
C_ASSERT( BLOCK_SIZE_BIN( 0 ) == 0 );
C_ASSERT( BLOCK_SIZE_BIN( 0x10 ) == 0 );
C_ASSERT( BLOCK_BIN_SIZE( 0 ) == BIN_SIZE_MIN_0 + 1 * BIN_SIZE_STEP_0 );
C_ASSERT( BLOCK_SIZE_BIN( 0x11 ) == 1 );
C_ASSERT( BLOCK_BIN_SIZE( 1 ) == BIN_SIZE_MIN_0 + 2 * BIN_SIZE_STEP_0 );
C_ASSERT( BLOCK_SIZE_BIN( BIN_SIZE_MAX ) == 0x7f );
C_ASSERT( BLOCK_BIN_SIZE( 0x7f ) == BIN_SIZE_MAX );
C_ASSERT( BLOCK_SIZE_BIN( BIN_SIZE_MAX + 1 ) == 0x80 );
C_ASSERT( BLOCK_BIN_SIZE( 0x80 ) == ~(SIZE_T)0 );
/* difference between block classes and all possible validation overhead must fit into block tail_size */
C_ASSERT( BIN_SIZE_STEP_7 + 3 * BLOCK_ALIGN <= FIELD_MAX( struct block, tail_size ) );
static BYTE affinity_mapping[] = {20,6,31,15,14,29,27,4,18,24,26,13,0,9,2,30,17,7,23,25,10,19,12,3,22,21,5,16,1,28,11,8};
static LONG next_thread_affinity;
/* a bin, tracking heap blocks of a certain size */
struct bin
{
/* counters for LFH activation */
LONG count_alloc;
LONG count_freed;
LONG enabled;
/* list of groups with free blocks */
SLIST_HEADER groups;
/* array of affinity reserved groups, interleaved with other bins to keep
* all pointers of the same affinity and different bin grouped together,
* and pointers of the same bin and different affinity away from each other,
* hopefully in separate cache lines.
*/
struct group **affinity_group_base;
};
static inline struct group **bin_get_affinity_group( struct bin *bin, BYTE affinity )
{
return bin->affinity_group_base + affinity * BLOCK_SIZE_BIN_COUNT;
}
struct heap
{ /* win32/win64 */
DWORD_PTR unknown1[2]; /* 0000/0000 */
DWORD ffeeffee; /* 0008/0010 */
DWORD auto_flags; /* 000c/0014 */
DWORD_PTR unknown2[7]; /* 0010/0018 */
DWORD unknown3[2]; /* 002c/0050 */
DWORD_PTR unknown4[3]; /* 0034/0058 */
DWORD flags; /* 0040/0070 */
DWORD force_flags; /* 0044/0074 */
/* end of the Windows 10 compatible struct layout */
LONG compat_info; /* HeapCompatibilityInformation / heap frontend type */
struct list entry; /* Entry in process heap list */
struct list subheap_list; /* Sub-heap list */
struct list large_list; /* Large blocks list */
SIZE_T grow_size; /* Size of next subheap for growing heap */
SIZE_T min_size; /* Minimum committed size */
DWORD magic; /* Magic number */
DWORD pending_pos; /* Position in pending free requests ring */
struct block **pending_free; /* Ring buffer for pending free requests */
RTL_CRITICAL_SECTION cs;
struct entry free_lists[FREE_LIST_COUNT];
struct bin *bins;
SUBHEAP subheap;
};
/* subheap must be last and aligned */
C_ASSERT( sizeof(struct heap) == offsetof(struct heap, subheap) + sizeof(SUBHEAP) );
C_ASSERT( sizeof(struct heap) % BLOCK_ALIGN == 0 );
C_ASSERT( offsetof(struct heap, subheap) <= REGION_ALIGN - 1 );
#define HEAP_MAGIC ((DWORD)('H' | ('E'<<8) | ('A'<<16) | ('P'<<24)))
#define HEAP_INITIAL_SIZE 0x10000
#define HEAP_INITIAL_GROW_SIZE 0x100000
#define HEAP_MAX_GROW_SIZE 0xfd0000
C_ASSERT( HEAP_MIN_LARGE_BLOCK_SIZE <= HEAP_INITIAL_GROW_SIZE );
#define MAX_FREE_PENDING 1024 /* max number of free requests to delay */
/* some undocumented flags (names are made up) */
#define HEAP_PRIVATE 0x00001000
#define HEAP_ADD_USER_INFO 0x00000100
#define HEAP_USER_FLAGS_MASK 0x00000f00
#define HEAP_PAGE_ALLOCS 0x01000000
#define HEAP_VALIDATE 0x10000000
#define HEAP_VALIDATE_ALL 0x20000000
#define HEAP_VALIDATE_PARAMS 0x40000000
#define HEAP_CHECKING_ENABLED 0x80000000
static struct heap *process_heap; /* main process heap */
static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block );
/* check if memory range a contains memory range b */
static inline BOOL contains( const void *a, SIZE_T a_size, const void *b, SIZE_T b_size )
{
const void *a_end = (char *)a + a_size, *b_end = (char *)b + b_size;
return a <= b && b <= b_end && b_end <= a_end;
}
static inline UINT block_get_flags( const struct block *block )
{
return block->block_flags;
}
static inline UINT block_get_type( const struct block *block )
{
return block->block_type;
}
static inline void block_set_base( struct block *block, const void *base )
{
const char *offset = ROUND_ADDR( block, REGION_ALIGN - 1 );
block->base_offset = (offset - (char *)base) / REGION_ALIGN;
}
static inline void block_set_type( struct block *block, UINT type )
{
block->block_type = type;
}
static inline SUBHEAP *block_get_subheap( const struct heap *heap, const struct block *block )
{
char *offset = ROUND_ADDR( block, REGION_ALIGN - 1 );
void *base = offset - (SIZE_T)block->base_offset * REGION_ALIGN;
if (base != (void *)heap) return base;
else return (SUBHEAP *)&heap->subheap;
}
static inline UINT block_get_overhead( const struct block *block )
{
if (block_get_flags( block ) & BLOCK_FLAG_FREE) return sizeof(*block) + sizeof(struct list);
return sizeof(*block) + block->tail_size;
}
/* return the size of a block, including its header */
static inline UINT block_get_size( const struct block *block )
{
UINT block_size = block->block_size;
if (block_get_flags( block ) & BLOCK_FLAG_FREE) block_size += (UINT)block->tail_size << 16;
return block_size * BLOCK_ALIGN;
}
static inline void block_set_size( struct block *block, UINT block_size )
{
block_size /= BLOCK_ALIGN;
if (block_get_flags( block ) & BLOCK_FLAG_FREE) block->tail_size = block_size >> 16;
block->block_size = block_size;
}
static inline void block_set_flags( struct block *block, BYTE clear, BYTE set )
{
UINT block_size = block_get_size( block );
block->block_flags &= ~clear;
block->block_flags |= set;
block_set_size( block, block_size );
}
static inline void *subheap_base( const SUBHEAP *subheap )
{
return ROUND_ADDR( subheap, REGION_ALIGN - 1 );
}
static inline SIZE_T subheap_overhead( const SUBHEAP *subheap )
{
return (char *)&subheap->block - (char *)subheap_base( subheap );
}
static inline SIZE_T subheap_size( const SUBHEAP *subheap )
{
return subheap->block_size + subheap_overhead( subheap );
}
static inline const void *subheap_commit_end( const SUBHEAP *subheap )
{
return (char *)(subheap + 1) + subheap->data_size;
}
static void subheap_set_bounds( SUBHEAP *subheap, char *commit_end, char *end )
{
subheap->block_size = end - (char *)&subheap->block;
subheap->data_size = commit_end - (char *)(subheap + 1);
}
static inline void *first_block( const SUBHEAP *subheap )
{
return (void *)&subheap->block;
}
static inline const void *last_block( const SUBHEAP *subheap )
{
return (char *)subheap_commit_end( subheap ) - sizeof(struct block);
}
static inline struct block *next_block( const SUBHEAP *subheap, const struct block *block )
{
const char *data = (char *)(block + 1), *next, *last = last_block( subheap );
next = (char *)block + block_get_size( block );
if (!contains( data, last - (char *)data, next, sizeof(*block) )) return NULL;
return (struct block *)next;
}
static inline BOOL check_subheap( const SUBHEAP *subheap, const struct heap *heap )
{
if (subheap->user_value != heap) return FALSE;
return contains( &subheap->block, subheap->block_size, subheap + 1, subheap->data_size );
}
static BOOL heap_validate( const struct heap *heap );
/* mark a block of memory as innacessible for debugging purposes */
static inline void valgrind_make_noaccess( void const *ptr, SIZE_T size )
{
#if defined(VALGRIND_MAKE_MEM_NOACCESS)
VALGRIND_DISCARD( VALGRIND_MAKE_MEM_NOACCESS( ptr, size ) );
#elif defined(VALGRIND_MAKE_NOACCESS)
VALGRIND_DISCARD( VALGRIND_MAKE_NOACCESS( ptr, size ) );
#endif
}
/* mark a block of memory as initialized for debugging purposes */
static inline void valgrind_make_readable( void const *ptr, SIZE_T size )
{
#if defined(VALGRIND_MAKE_MEM_DEFINED)
VALGRIND_DISCARD( VALGRIND_MAKE_MEM_DEFINED( ptr, size ) );
#elif defined(VALGRIND_MAKE_READABLE)
VALGRIND_DISCARD( VALGRIND_MAKE_READABLE( ptr, size ) );
#endif
}
/* mark a block of memory as uninitialized for debugging purposes */
static inline void valgrind_make_writable( void const *ptr, SIZE_T size )
{
#if defined(VALGRIND_MAKE_MEM_UNDEFINED)
VALGRIND_DISCARD( VALGRIND_MAKE_MEM_UNDEFINED( ptr, size ) );
#elif defined(VALGRIND_MAKE_WRITABLE)
VALGRIND_DISCARD( VALGRIND_MAKE_WRITABLE( ptr, size ) );
#endif
}
/* mark a block of memory as free for debugging purposes */
static inline void mark_block_free( void *ptr, SIZE_T size, DWORD flags )
{
if (flags & HEAP_FREE_CHECKING_ENABLED)
{
SIZE_T i;
for (i = 0; i < size / sizeof(DWORD); i++) ((DWORD *)ptr)[i] = BLOCK_FILL_FREE;
}
valgrind_make_noaccess( ptr, size );
}
/* mark a block of memory as a tail block */
static inline void mark_block_tail( struct block *block, DWORD flags )
{
char *tail = (char *)block + block_get_size( block ) - block->tail_size;
if (flags & HEAP_TAIL_CHECKING_ENABLED)
{
valgrind_make_writable( tail, BLOCK_ALIGN );
memset( tail, BLOCK_FILL_TAIL, BLOCK_ALIGN );
}
valgrind_make_noaccess( tail, BLOCK_ALIGN );
if (flags & HEAP_ADD_USER_INFO)
{
if (flags & HEAP_TAIL_CHECKING_ENABLED || RUNNING_ON_VALGRIND) tail += BLOCK_ALIGN;
valgrind_make_writable( tail, BLOCK_ALIGN );
memset( tail, 0, BLOCK_ALIGN );
}
}
/* initialize contents of a newly created block of memory */
static inline void initialize_block( struct block *block, SIZE_T old_size, SIZE_T size, DWORD flags )
{
char *data = (char *)(block + 1);
SIZE_T i;
if (size <= old_size) return;
if (flags & HEAP_ZERO_MEMORY)
{
valgrind_make_writable( data + old_size, size - old_size );
memset( data + old_size, 0, size - old_size );
}
else if (flags & HEAP_FREE_CHECKING_ENABLED)
{
valgrind_make_writable( data + old_size, size - old_size );
i = ROUND_SIZE( old_size, sizeof(DWORD) - 1 ) / sizeof(DWORD);
for (; i < size / sizeof(DWORD); ++i) ((DWORD *)data)[i] = BLOCK_FILL_USED;
}
}
/* notify that a new block of memory has been allocated for debugging purposes */
static inline void valgrind_notify_alloc( void const *ptr, SIZE_T size, BOOL init )
{
#ifdef VALGRIND_MALLOCLIKE_BLOCK
VALGRIND_MALLOCLIKE_BLOCK( ptr, size, 0, init );
#endif
}
/* notify that a block of memory has been freed for debugging purposes */
static inline void valgrind_notify_free( void const *ptr )
{
#ifdef VALGRIND_FREELIKE_BLOCK
VALGRIND_FREELIKE_BLOCK( ptr, 0 );
#endif
}
static inline void valgrind_notify_resize( void const *ptr, SIZE_T size_old, SIZE_T size_new )
{
#ifdef VALGRIND_RESIZEINPLACE_BLOCK
/* zero is not a valid size */
VALGRIND_RESIZEINPLACE_BLOCK( ptr, size_old ? size_old : 1, size_new ? size_new : 1, 0 );
#endif
}
static void valgrind_notify_free_all( SUBHEAP *subheap, const struct heap *heap )
{
#ifdef VALGRIND_FREELIKE_BLOCK
struct block *block;
if (!RUNNING_ON_VALGRIND) return;
if (!check_subheap( subheap, heap )) return;
for (block = first_block( subheap ); block; block = next_block( subheap, block ))
{
if (block_get_flags( block ) & BLOCK_FLAG_FREE) continue;
if (block_get_type( block ) == BLOCK_TYPE_USED) valgrind_notify_free( block + 1 );
}
#endif
}
/* get the memory protection type to use for a given heap */
static inline ULONG get_protection_type( DWORD flags )
{
return (flags & HEAP_CREATE_ENABLE_EXECUTE) ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE;
}
static RTL_CRITICAL_SECTION_DEBUG process_heap_cs_debug =
{
0, 0, NULL, /* will be set later */
{ &process_heap_cs_debug.ProcessLocksList, &process_heap_cs_debug.ProcessLocksList },
0, 0, { (DWORD_PTR)(__FILE__ ": main process heap section") }
};
static inline ULONG heap_get_flags( const struct heap *heap, ULONG flags )
{
if (flags & (HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED)) flags |= HEAP_CHECKING_ENABLED;
flags &= HEAP_GENERATE_EXCEPTIONS | HEAP_NO_SERIALIZE | HEAP_ZERO_MEMORY | HEAP_REALLOC_IN_PLACE_ONLY | HEAP_CHECKING_ENABLED | HEAP_USER_FLAGS_MASK;
return heap->flags | flags;
}
static inline void heap_lock( struct heap *heap, ULONG flags )
{
if (flags & HEAP_NO_SERIALIZE) return;
RtlEnterCriticalSection( &heap->cs );
}
static inline void heap_unlock( struct heap *heap, ULONG flags )
{
if (flags & HEAP_NO_SERIALIZE) return;
RtlLeaveCriticalSection( &heap->cs );
}
static void heap_set_status( const struct heap *heap, ULONG flags, NTSTATUS status )
{
if (status == STATUS_NO_MEMORY && (flags & HEAP_GENERATE_EXCEPTIONS)) RtlRaiseStatus( status );
if (status) RtlSetLastWin32ErrorAndNtStatusFromNtStatus( status );
}
static SIZE_T get_free_list_block_size( unsigned int index )
{
DWORD log = index >> FREE_LIST_LINEAR_BITS;
DWORD linear = index & FREE_LIST_LINEAR_MASK;
if (log == 0) return index * BLOCK_ALIGN;
return (((1 << FREE_LIST_LINEAR_BITS) + linear) << (log - 1)) * BLOCK_ALIGN;
}
/*
* Given a size, return its index in the block size list for freelists.
*
* With FREE_LIST_LINEAR_BITS=2, the list looks like this
* (with respect to size / BLOCK_ALIGN):
* 0,
* 1, 2, 3, 4, 5, 6, 7, 8,
* 10, 12, 14, 16, 20, 24, 28, 32,
* 40, 48, 56, 64, 80, 96, 112, 128,
* 160, 192, 224, 256, 320, 384, 448, 512,
* ...
*/
static unsigned int get_free_list_index( SIZE_T block_size )
{
DWORD bit, log, linear;
if (block_size > get_free_list_block_size( FREE_LIST_COUNT - 1 ))
return FREE_LIST_COUNT - 1;
block_size /= BLOCK_ALIGN;
/* find the highest bit */
if (!BitScanReverse( &bit, block_size ) || bit < FREE_LIST_LINEAR_BITS)
{
/* for small values, the index is same as block_size. */
log = 0;
linear = block_size;
}
else
{
/* the highest bit is always set, ignore it and encode the next FREE_LIST_LINEAR_BITS bits
* as a linear scale, combined with the shift as a log scale, in the free list index. */
log = bit - FREE_LIST_LINEAR_BITS + 1;
linear = (block_size >> (bit - FREE_LIST_LINEAR_BITS)) & FREE_LIST_LINEAR_MASK;
}
return (log << FREE_LIST_LINEAR_BITS) + linear;
}
/* locate a free list entry of the appropriate size */
static inline struct entry *find_free_list( struct heap *heap, SIZE_T block_size, BOOL last )
{
unsigned int index = get_free_list_index( block_size );
if (last && ++index == FREE_LIST_COUNT) index = 0;
return &heap->free_lists[index];
}
static void heap_dump( const struct heap *heap )
{
const struct block *block;
const ARENA_LARGE *large;
const SUBHEAP *subheap;
unsigned int i;
TRACE( "heap: %p\n", heap );
TRACE( " next %p\n", LIST_ENTRY( heap->entry.next, struct heap, entry ) );
TRACE( " bins:\n" );
for (i = 0; heap->bins && i < BLOCK_SIZE_BIN_COUNT; i++)
{
const struct bin *bin = heap->bins + i;
ULONG alloc = ReadNoFence( &bin->count_alloc ), freed = ReadNoFence( &bin->count_freed );
if (!alloc && !freed) continue;
TRACE( " %3u: size %#4Ix, alloc %ld, freed %ld, enabled %lu\n", i, BLOCK_BIN_SIZE( i ),
alloc, freed, ReadNoFence( &bin->enabled ) );
}
TRACE( " free_lists: %p\n", heap->free_lists );
for (i = 0; i < FREE_LIST_COUNT; i++)
TRACE( " %p: size %#8Ix, prev %p, next %p\n", heap->free_lists + i, get_free_list_block_size( i ),
LIST_ENTRY( heap->free_lists[i].entry.prev, struct entry, entry ),
LIST_ENTRY( heap->free_lists[i].entry.next, struct entry, entry ) );
TRACE( " subheaps: %p\n", &heap->subheap_list );
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
{
SIZE_T free_size = 0, used_size = 0, overhead = 0;
const char *base = subheap_base( subheap );
TRACE( " %p: base %p first %p last %p end %p\n", subheap, base, first_block( subheap ),
last_block( subheap ), base + subheap_size( subheap ) );
if (!check_subheap( subheap, heap )) return;
overhead += subheap_overhead( subheap );
for (block = first_block( subheap ); block; block = next_block( subheap, block ))
{
if (block_get_flags( block ) & BLOCK_FLAG_FREE)
{
TRACE( " %p: (free) type %#10x, size %#8x, flags %#4x, prev %p, next %p\n", block,
block_get_type( block ), block_get_size( block ), block_get_flags( block ),
LIST_ENTRY( ((struct entry *)block)->entry.prev, struct entry, entry ),
LIST_ENTRY( ((struct entry *)block)->entry.next, struct entry, entry ) );
overhead += block_get_overhead( block );
free_size += block_get_size( block ) - block_get_overhead( block );
}
else
{
TRACE( " %p: (used) type %#10x, size %#8x, flags %#4x, unused %#4x", block,
block_get_type( block ), block_get_size( block ), block_get_flags( block ),
block->tail_size );
if (!(block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)) TRACE( "\n" );
else TRACE( ", back %p\n", *((struct block **)block - 1) );
overhead += block_get_overhead( block );
used_size += block_get_size( block ) - block_get_overhead( block );
}
}
TRACE( " total %#Ix, used %#Ix, free %#Ix, overhead %#Ix (%Iu%%)\n", used_size + free_size + overhead,
used_size, free_size, overhead, (overhead * 100) / subheap_size( subheap ) );
}
TRACE( " large blocks: %p\n", &heap->large_list );
LIST_FOR_EACH_ENTRY( large, &heap->large_list, ARENA_LARGE, entry )
{
TRACE( " %p: (large) type %#10x, size %#8x, flags %#4x, total_size %#10Ix, alloc_size %#10Ix, prev %p, next %p\n",
large, block_get_type( &large->block ), block_get_size( &large->block ), block_get_flags( &large->block ), large->block_size,
large->data_size, LIST_ENTRY( large->entry.prev, ARENA_LARGE, entry ), LIST_ENTRY( large->entry.next, ARENA_LARGE, entry ) );
}
if (heap->pending_free)
{
TRACE( " pending blocks: %p\n", heap->pending_free );
for (i = 0; i < MAX_FREE_PENDING; ++i)
{
if (!(block = heap->pending_free[i])) break;
TRACE( " %c%p: (pend) type %#10x, size %#8x, flags %#4x, unused %#4x", i == heap->pending_pos ? '*' : ' ',
block, block_get_type( block ), block_get_size( block ), block_get_flags( block ), block->tail_size );
if (!(block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)) TRACE( "\n" );
else TRACE( ", back %p\n", *((struct block **)block - 1) );
}
}
}
static const char *debugstr_heap_entry( struct rtl_heap_entry *entry )
{
const char *str = wine_dbg_sprintf( "data %p, size %#Ix, overhead %#x, region %#x, flags %#x", entry->lpData,
entry->cbData, entry->cbOverhead, entry->iRegionIndex, entry->wFlags );
if (!(entry->wFlags & RTL_HEAP_ENTRY_REGION)) return str;
return wine_dbg_sprintf( "%s, commit %#lx, uncommit %#lx, first %p, last %p", str, entry->Region.dwCommittedSize,
entry->Region.dwUnCommittedSize, entry->Region.lpFirstBlock, entry->Region.lpLastBlock );
}
static struct heap *unsafe_heap_from_handle( HANDLE handle, ULONG flags, ULONG *heap_flags )
{
struct heap *heap = handle;
BOOL valid = TRUE;
if (!heap || (heap->magic != HEAP_MAGIC))
{
ERR( "Invalid handle %p!\n", handle );
return NULL;
}
if (heap->flags & HEAP_VALIDATE_ALL)
{
heap_lock( heap, 0 );
valid = heap_validate( heap );
heap_unlock( heap, 0 );
if (!valid && TRACE_ON(heap))
{
heap_dump( heap );
assert( FALSE );
}
}
*heap_flags = heap_get_flags( heap, flags );
return valid ? heap : NULL;
}
static SUBHEAP *find_subheap( const struct heap *heap, const struct block *block, BOOL heap_walk )
{
SUBHEAP *subheap;
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
{
SIZE_T blocks_size = (char *)last_block( subheap ) - (char *)first_block( subheap );
if (!check_subheap( subheap, heap )) return NULL;
if (contains( first_block( subheap ), blocks_size, block, sizeof(*block) )) return subheap;
/* outside of blocks region, possible corruption or heap_walk */
if (contains( subheap_base( subheap ), subheap_size( subheap ), block, 1 )) return heap_walk ? subheap : NULL;
}
return NULL;
}
static inline BOOL subheap_commit( const struct heap *heap, SUBHEAP *subheap, const struct block *block, SIZE_T block_size )
{
const char *end = (char *)subheap_base( subheap ) + subheap_size( subheap ), *commit_end;
SIZE_T size;
void *addr;
commit_end = (char *)block + block_size + sizeof(struct entry);
commit_end = ROUND_ADDR( (char *)commit_end + REGION_ALIGN - 1, REGION_ALIGN - 1 );
if (commit_end > end) commit_end = end;
if (commit_end <= (char *)subheap_commit_end( subheap )) return TRUE;
addr = (void *)subheap_commit_end( subheap );
size = commit_end - (char *)addr;
if (NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, &size, MEM_COMMIT,
get_protection_type( heap->flags ) ))
{
WARN( "Could not commit %#Ix bytes at %p for heap %p\n", size, addr, heap );
return FALSE;
}
subheap->data_size = (char *)commit_end - (char *)(subheap + 1);
return TRUE;
}
static inline BOOL subheap_decommit( const struct heap *heap, SUBHEAP *subheap, const void *commit_end )
{
char *base = subheap_base( subheap );
SIZE_T size;
void *addr;
commit_end = ROUND_ADDR( (char *)commit_end + REGION_ALIGN - 1, REGION_ALIGN - 1 );
if (subheap == &heap->subheap) commit_end = max( (char *)commit_end, (char *)base + heap->min_size );
if (commit_end >= subheap_commit_end( subheap )) return TRUE;
size = (char *)subheap_commit_end( subheap ) - (char *)commit_end;
addr = (void *)commit_end;
if (NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_DECOMMIT ))
{
WARN( "Could not decommit %#Ix bytes at %p for heap %p\n", size, addr, heap );
return FALSE;
}
subheap->data_size = (char *)commit_end - (char *)(subheap + 1);
return TRUE;
}
static void block_init_free( struct block *block, ULONG flags, SUBHEAP *subheap, SIZE_T block_size )
{
const char *end = (char *)block + block_size, *commit_end = subheap_commit_end( subheap );
struct entry *entry = (struct entry *)block;
valgrind_make_writable( block, sizeof(*entry) );
block_set_type( block, BLOCK_TYPE_FREE );
block_set_base( block, subheap_base( subheap ) );
block_set_flags( block, ~0, BLOCK_FLAG_FREE );
block_set_size( block, block_size );
/* If debugging, erase the freed block content */
if (end > commit_end) end = commit_end;
if (end > (char *)(entry + 1)) mark_block_free( entry + 1, end - (char *)(entry + 1), flags );
}
static void insert_free_block( struct heap *heap, ULONG flags, SUBHEAP *subheap, struct block *block )
{
struct entry *entry = (struct entry *)block, *list;
struct block *next;
if ((next = next_block( subheap, block )))
{
/* set the next block PREV_FREE flag and back pointer */
block_set_flags( next, 0, BLOCK_FLAG_PREV_FREE );
valgrind_make_writable( (struct block **)next - 1, sizeof(struct block *) );
*((struct block **)next - 1) = block;
}
list = find_free_list( heap, block_get_size( block ), !next );
if (!next) list_add_before( &list->entry, &entry->entry );
else list_add_after( &list->entry, &entry->entry );
}
static struct block *heap_delay_free( struct heap *heap, ULONG flags, struct block *block )
{
struct block *tmp;
if (!heap->pending_free) return block;
block_set_type( block, BLOCK_TYPE_DEAD );
mark_block_free( block + 1, block_get_size( block ) - sizeof(*block), flags );
heap_lock( heap, flags );
tmp = heap->pending_free[heap->pending_pos];
heap->pending_free[heap->pending_pos] = block;
heap->pending_pos = (heap->pending_pos + 1) % MAX_FREE_PENDING;
heap_unlock( heap, flags );
return tmp;
}
static NTSTATUS heap_free_block( struct heap *heap, ULONG flags, struct block *block )
{
SUBHEAP *subheap = block_get_subheap( heap, block );
SIZE_T block_size = block_get_size( block );
struct entry *entry;
struct block *next;
if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE))
{
/* merge with next block if it is free */
entry = (struct entry *)next;
block_size += block_get_size( &entry->block );
list_remove( &entry->entry );
next = next_block( subheap, next );
}
if (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)
{
/* merge with previous block if it is free */
entry = *((struct entry **)block - 1);
block_size += block_get_size( &entry->block );
list_remove( &entry->entry );
block = &entry->block;
}
if (block == first_block( subheap ) && !next && subheap != &heap->subheap)
{
/* free the subheap if it's empty and not the main one */
void *addr = subheap_base( subheap );
SIZE_T size = 0;
list_remove( &subheap->entry );
NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
return STATUS_SUCCESS;
}
block_init_free( block, flags, subheap, block_size );
insert_free_block( heap, flags, subheap, block );
/* keep room for a full committed block as hysteresis */
if (!next) subheap_decommit( heap, subheap, (char *)((struct entry *)block + 1) + REGION_ALIGN );
return STATUS_SUCCESS;
}
static struct block *split_block( struct heap *heap, ULONG flags, struct block *block,
SIZE_T old_block_size, SIZE_T block_size )
{
SUBHEAP *subheap = block_get_subheap( heap, block );
if (old_block_size >= block_size + HEAP_MIN_BLOCK_SIZE)
{
block_set_size( block, block_size );
return next_block( subheap, block );
}
block_set_size( block, old_block_size );
return NULL;
}
static void *allocate_region( struct heap *heap, ULONG flags, SIZE_T *region_size, SIZE_T *commit_size )
{
void *addr = NULL;
NTSTATUS status;
if (heap && !(flags & HEAP_GROWABLE))
{
WARN( "Heap %p isn't growable, cannot allocate %#Ix bytes\n", heap, *region_size );
return NULL;
}
/* allocate the memory block */
if ((status = NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, region_size, MEM_RESERVE,
get_protection_type( flags ) )))
{
WARN( "Could not allocate %#Ix bytes, status %#lx\n", *region_size, status );
return NULL;
}
if ((status = NtAllocateVirtualMemory( NtCurrentProcess(), &addr, 0, commit_size, MEM_COMMIT,
get_protection_type( flags ) )))
{
WARN( "Could not commit %#Ix bytes, status %#lx\n", *commit_size, status );
return NULL;
}
return addr;
}
static NTSTATUS heap_allocate_large( struct heap *heap, ULONG flags, SIZE_T block_size,
SIZE_T size, void **ret )
{
ARENA_LARGE *arena;
SIZE_T total_size = ROUND_SIZE( sizeof(*arena) + size, REGION_ALIGN - 1 );
struct block *block;
if (total_size < size) return STATUS_NO_MEMORY; /* overflow */
if (!(arena = allocate_region( heap, flags, &total_size, &total_size ))) return STATUS_NO_MEMORY;
block = &arena->block;
arena->data_size = size;
arena->block_size = (char *)arena + total_size - (char *)block;
block_set_type( block, BLOCK_TYPE_LARGE );
block_set_base( block, arena );
block_set_flags( block, ~0, BLOCK_FLAG_LARGE | BLOCK_USER_FLAGS( flags ) );
block_set_size( block, 0 );
heap_lock( heap, flags );
list_add_tail( &heap->large_list, &arena->entry );
heap_unlock( heap, flags );
valgrind_make_noaccess( (char *)block + sizeof(*block) + arena->data_size,
arena->block_size - sizeof(*block) - arena->data_size );
*ret = block + 1;
return STATUS_SUCCESS;
}
static NTSTATUS heap_free_large( struct heap *heap, ULONG flags, struct block *block )
{
ARENA_LARGE *arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
LPVOID address = arena;
SIZE_T size = 0;
heap_lock( heap, flags );
list_remove( &arena->entry );
heap_unlock( heap, flags );
return NtFreeVirtualMemory( NtCurrentProcess(), &address, &size, MEM_RELEASE );
}
/***********************************************************************
* find_large_block
*/
static BOOL find_large_block( const struct heap *heap, const struct block *block )
{
ARENA_LARGE *arena;
LIST_FOR_EACH_ENTRY( arena, &heap->large_list, ARENA_LARGE, entry )
if (block == &arena->block) return TRUE;
return FALSE;
}
static BOOL validate_large_block( const struct heap *heap, const struct block *block )
{
const ARENA_LARGE *arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
const char *err = NULL;
if (ROUND_ADDR( block, REGION_ALIGN - 1 ) != arena)
err = "invalid block BLOCK_ALIGN";
else if (block_get_size( block ))
err = "invalid block size";
else if (!(block_get_flags( block ) & BLOCK_FLAG_LARGE))
err = "invalid block flags";
else if (block_get_type( block ) != BLOCK_TYPE_LARGE)
err = "invalid block type";
else if (!contains( block, arena->block_size, block + 1, arena->data_size ))
err = "invalid block size";
if (err)
{
ERR( "heap %p, block %p: %s\n", heap, block, err );
if (TRACE_ON(heap)) heap_dump( heap );
}
return !err;
}
static SUBHEAP *create_subheap( struct heap *heap, DWORD flags, SIZE_T total_size, SIZE_T commit_size )
{
SIZE_T block_size;
SUBHEAP *subheap;
commit_size = ROUND_SIZE( max( commit_size, REGION_ALIGN ), REGION_ALIGN - 1 );
total_size = min( max( commit_size, total_size ), 0xffff0000 ); /* don't allow a heap larger than 4GB */
if (!(subheap = allocate_region( heap, flags, &total_size, &commit_size ))) return NULL;
subheap->user_value = heap;
subheap_set_bounds( subheap, (char *)subheap + commit_size, (char *)subheap + total_size );
block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 );
block_init_free( first_block( subheap ), flags, subheap, block_size );
list_add_head( &heap->subheap_list, &subheap->entry );
return subheap;
}
static struct block *find_free_block( struct heap *heap, ULONG flags, SIZE_T block_size )
{
struct list *ptr = &find_free_list( heap, block_size, FALSE )->entry;
struct entry *entry;
struct block *block;
SIZE_T total_size;
SUBHEAP *subheap;
/* Find a suitable free list, and in it find a block large enough */
while ((ptr = list_next( &heap->free_lists[0].entry, ptr )))
{
entry = LIST_ENTRY( ptr, struct entry, entry );
block = &entry->block;
if (block_get_flags( block ) == BLOCK_FLAG_FREE_LINK) continue;
if (block_get_size( block ) >= block_size)
{
if (!subheap_commit( heap, block_get_subheap( heap, block ), block, block_size )) return NULL;
list_remove( &entry->entry );
return block;
}
}
/* make sure we can fit the block and a free entry at the end */
total_size = sizeof(SUBHEAP) + block_size + sizeof(struct entry);
if (total_size < block_size) return NULL; /* overflow */
if ((subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size )))
{
heap->grow_size = min( heap->grow_size * 2, HEAP_MAX_GROW_SIZE );
}
else while (!subheap) /* shrink the grow size again if we are running out of space */
{
if (heap->grow_size <= total_size || heap->grow_size <= 4 * 1024 * 1024) return NULL;
heap->grow_size /= 2;
subheap = create_subheap( heap, flags, max( heap->grow_size, total_size ), total_size );
}
TRACE( "created new sub-heap %p of %#Ix bytes for heap %p\n", subheap, subheap_size( subheap ), heap );
return first_block( subheap );
}
static BOOL is_valid_free_block( const struct heap *heap, const struct block *block )
{
const SUBHEAP *subheap;
unsigned int i;
if ((subheap = find_subheap( heap, block, FALSE ))) return TRUE;
for (i = 0; i < FREE_LIST_COUNT; i++) if (block == &heap->free_lists[i].block) return TRUE;
return FALSE;
}
static BOOL validate_free_block( const struct heap *heap, const SUBHEAP *subheap, const struct block *block )
{
const char *err = NULL, *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
const struct entry *entry = (struct entry *)block;
const struct block *prev, *next;
DWORD flags = heap->flags;
if ((ULONG_PTR)(block + 1) % BLOCK_ALIGN)
err = "invalid block BLOCK_ALIGN";
else if (block_get_type( block ) != BLOCK_TYPE_FREE)
err = "invalid block header";
else if (!(block_get_flags( block ) & BLOCK_FLAG_FREE) || (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE))
err = "invalid block flags";
else if (!contains( base, subheap_size( subheap ), block, block_get_size( block ) ))
err = "invalid block size";
else if (!is_valid_free_block( heap, (next = &LIST_ENTRY( entry->entry.next, struct entry, entry )->block) ))
err = "invalid next free block pointer";
else if (!(block_get_flags( next ) & BLOCK_FLAG_FREE) || block_get_type( next ) != BLOCK_TYPE_FREE)
err = "invalid next free block header";
else if (!is_valid_free_block( heap, (prev = &LIST_ENTRY( entry->entry.prev, struct entry, entry )->block) ))
err = "invalid previous free block pointer";
else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE)
err = "invalid previous free block header";
else if ((next = next_block( subheap, block )))
{
if (!(block_get_flags( next ) & BLOCK_FLAG_PREV_FREE))
err = "invalid next block flags";
if (*((struct block **)next - 1) != block)
err = "invalid next block back pointer";
}
if (!err && (flags & HEAP_FREE_CHECKING_ENABLED))
{
const char *ptr = (char *)(entry + 1), *end = (char *)block + block_get_size( block );
if (next) end -= sizeof(struct block *);
if (end > commit_end) end = commit_end;
while (!err && ptr < end)
{
if (*(DWORD *)ptr != BLOCK_FILL_FREE) err = "free block overwritten";
ptr += sizeof(DWORD);
}
}
if (err)
{
ERR( "heap %p, block %p: %s\n", heap, block, err );
if (TRACE_ON(heap)) heap_dump( heap );
}
return !err;
}
static BOOL validate_used_block( const struct heap *heap, const SUBHEAP *subheap, const struct block *block,
unsigned int expected_block_type )
{
const char *err = NULL, *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
DWORD flags = heap->flags;
const struct block *next;
int i;
if ((ULONG_PTR)(block + 1) % BLOCK_ALIGN)
err = "invalid block BLOCK_ALIGN";
else if (block_get_type( block ) != BLOCK_TYPE_USED && block_get_type( block ) != BLOCK_TYPE_DEAD)
err = "invalid block header";
else if (expected_block_type && block_get_type( block ) != expected_block_type)
err = "invalid block type";
else if (block_get_flags( block ) & BLOCK_FLAG_FREE)
err = "invalid block flags";
else if (!contains( base, commit_end - base, block, block_get_size( block ) ))
err = "invalid block size";
else if (block->tail_size > block_get_size( block ) - sizeof(*block))
err = "invalid block unused size";
else if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_PREV_FREE) &&
/* LFH blocks do not use BLOCK_FLAG_PREV_FREE or back pointer */
!(block_get_flags( block ) & BLOCK_FLAG_LFH))
err = "invalid next block flags";
else if (block_get_flags( block ) & BLOCK_FLAG_PREV_FREE)
{
const struct block *prev = *((struct block **)block - 1);
if (!is_valid_free_block( heap, prev ))
err = "invalid previous block pointer";
else if (!(block_get_flags( prev ) & BLOCK_FLAG_FREE) || block_get_type( prev ) != BLOCK_TYPE_FREE)
err = "invalid previous block flags";
if ((char *)prev + block_get_size( prev ) != (char *)block)
err = "invalid previous block size";
}
if (!err && block_get_type( block ) == BLOCK_TYPE_DEAD)
{
const char *ptr = (char *)(block + 1), *end = (char *)block + block_get_size( block );
while (!err && ptr < end)
{
if (*(DWORD *)ptr != BLOCK_FILL_FREE) err = "free block overwritten";
ptr += sizeof(DWORD);
}
}
else if (!err && (flags & HEAP_TAIL_CHECKING_ENABLED))
{
const unsigned char *tail = (unsigned char *)block + block_get_size( block ) - block->tail_size;
for (i = 0; !err && i < BLOCK_ALIGN; i++) if (tail[i] != BLOCK_FILL_TAIL) err = "invalid block tail";
}
if (err)
{
ERR( "heap %p, block %p: %s\n", heap, block, err );
if (TRACE_ON(heap)) heap_dump( heap );
}
return !err;
}
static BOOL heap_validate_ptr( const struct heap *heap, const void *ptr )
{
const struct block *block = (struct block *)ptr - 1;
const SUBHEAP *subheap;
if (!(subheap = find_subheap( heap, block, FALSE )))
{
if (!find_large_block( heap, block ))
{
WARN("heap %p, ptr %p: block region not found\n", heap, ptr );
return FALSE;
}
return validate_large_block( heap, block );
}
return validate_used_block( heap, subheap, block, BLOCK_TYPE_USED );
}
static BOOL heap_validate( const struct heap *heap )
{
const ARENA_LARGE *large_arena;
const struct block *block;
const SUBHEAP *subheap;
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
{
if (!check_subheap( subheap, heap ))
{
ERR( "heap %p, subheap %p corrupted sizes or user_value\n", heap, subheap );
if (TRACE_ON(heap)) heap_dump( heap );
return FALSE;
}
for (block = first_block( subheap ); block; block = next_block( subheap, block ))
{
if (block_get_flags( block ) & BLOCK_FLAG_FREE)
{
if (!validate_free_block( heap, subheap, block )) return FALSE;
}
else
{
if (!validate_used_block( heap, subheap, block, 0 )) return FALSE;
}
}
}
if (heap->pending_free)
{
unsigned int i;
for (i = 0; i < MAX_FREE_PENDING; i++)
{
if (!(block = heap->pending_free[i])) break;
subheap = find_subheap( heap, block, FALSE );
if (!subheap)
{
ERR( "heap %p: cannot find valid subheap for delayed freed block %p\n", heap, block );
if (TRACE_ON(heap)) heap_dump( heap );
return FALSE;
}
if (!validate_used_block( heap, subheap, block, BLOCK_TYPE_DEAD )) return FALSE;
}
for (; i < MAX_FREE_PENDING; i++)
{
if ((block = heap->pending_free[i]))
{
ERR( "heap %p: unexpected delayed freed block %p at slot %u\n", heap, block, i );
if (TRACE_ON(heap)) heap_dump( heap );
return FALSE;
}
}
}
LIST_FOR_EACH_ENTRY( large_arena, &heap->large_list, ARENA_LARGE, entry )
if (!validate_large_block( heap, &large_arena->block )) return FALSE;
return TRUE;
}
static inline struct block *unsafe_block_from_ptr( struct heap *heap, ULONG flags, const void *ptr )
{
struct block *block = (struct block *)ptr - 1;
const char *err = NULL;
SUBHEAP *subheap;
if (flags & HEAP_VALIDATE)
{
heap_lock( heap, flags );
if (!heap_validate_ptr( heap, ptr )) block = NULL;
heap_unlock( heap, flags );
return block;
}
if ((ULONG_PTR)ptr % BLOCK_ALIGN)
err = "invalid ptr alignment";
else if (block_get_type( block ) == BLOCK_TYPE_DEAD)
err = "delayed freed block";
else if (block_get_type( block ) == BLOCK_TYPE_FREE)
err = "already freed block";
else if (block_get_flags( block ) & BLOCK_FLAG_LFH)
{
/* LFH block base_offset points to the group, not the subheap */
if (block_get_type( block ) != BLOCK_TYPE_USED)
err = "invalid block type";
}
else if ((subheap = block_get_subheap( heap, block )) >= (SUBHEAP *)block)
err = "invalid base offset";
else if (block_get_type( block ) == BLOCK_TYPE_USED)
{
const char *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap );
if (subheap->user_value != heap) err = "mismatching heap";
else if (!contains( base, commit_end - base, block, block_get_size( block ) )) err = "invalid block size";
}
else if (block_get_type( block ) == BLOCK_TYPE_LARGE)
{
ARENA_LARGE *large = subheap_base( subheap );
if (block != &large->block) err = "invalid large block";
}
else
{
err = "invalid block type";
}
if (err) WARN( "heap %p, block %p: %s\n", heap, block, err );
return err ? NULL : block;
}
static DWORD heap_flags_from_global_flag( DWORD flag )
{
DWORD ret = 0;
if (flag & FLG_HEAP_ENABLE_TAIL_CHECK)
ret |= HEAP_TAIL_CHECKING_ENABLED;
if (flag & FLG_HEAP_ENABLE_FREE_CHECK)
ret |= HEAP_FREE_CHECKING_ENABLED;
if (flag & FLG_HEAP_VALIDATE_PARAMETERS)
ret |= HEAP_VALIDATE_PARAMS | HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED;
if (flag & FLG_HEAP_VALIDATE_ALL)
ret |= HEAP_VALIDATE_ALL | HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED;
if (flag & FLG_HEAP_DISABLE_COALESCING)
ret |= HEAP_DISABLE_COALESCE_ON_FREE;
if (flag & FLG_HEAP_PAGE_ALLOCS)
ret |= HEAP_PAGE_ALLOCS;
return ret;
}
/***********************************************************************
* heap_set_debug_flags
*/
static void heap_set_debug_flags( HANDLE handle )
{
ULONG global_flags = RtlGetNtGlobalFlags();
DWORD dummy, flags, force_flags;
struct heap *heap;
if (TRACE_ON(heap)) global_flags |= FLG_HEAP_VALIDATE_ALL;
if (WARN_ON(heap)) global_flags |= FLG_HEAP_VALIDATE_PARAMETERS;
heap = unsafe_heap_from_handle( handle, 0, &dummy );
flags = heap_flags_from_global_flag( global_flags );
force_flags = (heap->flags | flags) & ~(HEAP_SHARED|HEAP_DISABLE_COALESCE_ON_FREE);
if (global_flags & FLG_HEAP_ENABLE_TAGGING) flags |= HEAP_SHARED;
if (!(global_flags & FLG_HEAP_PAGE_ALLOCS)) force_flags &= ~(HEAP_GROWABLE|HEAP_PRIVATE);
if (RUNNING_ON_VALGRIND) flags = 0; /* no sense in validating since Valgrind catches accesses */
heap->flags |= flags;
heap->force_flags |= force_flags;
if (flags & (HEAP_FREE_CHECKING_ENABLED | HEAP_TAIL_CHECKING_ENABLED)) /* fix existing blocks */
{
struct block *block;
SUBHEAP *subheap;
LIST_FOR_EACH_ENTRY( subheap, &heap->subheap_list, SUBHEAP, entry )
{
const char *commit_end = subheap_commit_end( subheap );
if (!check_subheap( subheap, heap )) break;
for (block = first_block( subheap ); block; block = next_block( subheap, block ))
{
if (block_get_flags( block ) & BLOCK_FLAG_FREE)
{
char *data = (char *)block + block_get_overhead( block ), *end = (char *)block + block_get_size( block );
if (next_block( subheap, block )) end -= sizeof(struct block *);
if (end > commit_end) mark_block_free( data, commit_end - data, flags );
else mark_block_free( data, end - data, flags );
}
else
{
if (block_get_type( block ) == BLOCK_TYPE_DEAD) mark_block_free( block + 1, block_get_size( block ) - sizeof(*block), flags );
else mark_block_tail( block, flags );
}
}
}
}
if ((heap->flags & HEAP_GROWABLE) && !heap->pending_free &&
((flags & HEAP_FREE_CHECKING_ENABLED) || RUNNING_ON_VALGRIND))
{
heap->pending_free = RtlAllocateHeap( handle, HEAP_ZERO_MEMORY,
MAX_FREE_PENDING * sizeof(*heap->pending_free) );
heap->pending_pos = 0;
}
}
/***********************************************************************
* RtlCreateHeap (NTDLL.@)
*
* Create a new Heap.
*
* PARAMS
* flags [I] HEAP_ flags from "winnt.h"
* addr [I] Desired base address
* totalSize [I] Total size of the heap, or 0 for a growable heap
* commitSize [I] Amount of heap space to commit
* unknown [I] Not yet understood
* definition [I] Heap definition
*
* RETURNS
* Success: A HANDLE to the newly created heap.
* Failure: a NULL HANDLE.
*/
HANDLE WINAPI RtlCreateHeap( ULONG flags, void *addr, SIZE_T total_size, SIZE_T commit_size,
void *unknown, RTL_HEAP_DEFINITION *definition )
{
struct entry *entry;
struct heap *heap;
SIZE_T block_size;
SUBHEAP *subheap;
unsigned int i;
TRACE( "flags %#lx, addr %p, total_size %#Ix, commit_size %#Ix, unknown %p, definition %p\n",
flags, addr, total_size, commit_size, unknown, definition );
flags &= ~(HEAP_TAIL_CHECKING_ENABLED|HEAP_FREE_CHECKING_ENABLED);
if (process_heap) flags |= HEAP_PRIVATE;
if (!process_heap || !total_size || (flags & HEAP_SHARED)) flags |= HEAP_GROWABLE;
if (!total_size) total_size = commit_size + HEAP_INITIAL_SIZE;
if (!(heap = addr))
{
if (!commit_size) commit_size = REGION_ALIGN;
total_size = min( max( total_size, commit_size ), 0xffff0000 ); /* don't allow a heap larger than 4GB */
commit_size = min( total_size, ROUND_SIZE( commit_size, REGION_ALIGN - 1 ) );
if (!(heap = allocate_region( NULL, flags, &total_size, &commit_size ))) return 0;
}
heap->ffeeffee = 0xffeeffee;
heap->auto_flags = (flags & HEAP_GROWABLE);
heap->flags = (flags & ~HEAP_SHARED);
heap->compat_info = HEAP_STD;
heap->magic = HEAP_MAGIC;
heap->grow_size = HEAP_INITIAL_GROW_SIZE;
heap->min_size = commit_size;
list_init( &heap->subheap_list );
list_init( &heap->large_list );
list_init( &heap->free_lists[0].entry );
for (i = 0, entry = heap->free_lists; i < FREE_LIST_COUNT; i++, entry++)
{
block_set_flags( &entry->block, ~0, BLOCK_FLAG_FREE_LINK );
block_set_size( &entry->block, 0 );
block_set_type( &entry->block, BLOCK_TYPE_FREE );
block_set_base( &entry->block, heap );
if (i) list_add_after( &entry[-1].entry, &entry->entry );
}
if (!process_heap) /* do it by hand to avoid memory allocations */
{
heap->cs.DebugInfo = &process_heap_cs_debug;
heap->cs.LockCount = -1;
heap->cs.RecursionCount = 0;
heap->cs.OwningThread = 0;
heap->cs.LockSemaphore = 0;
heap->cs.SpinCount = 0;
process_heap_cs_debug.CriticalSection = &heap->cs;
}
else
{
RtlInitializeCriticalSection( &heap->cs );
heap->cs.DebugInfo->Spare[0] = (DWORD_PTR)(__FILE__ ": heap.cs");
}
subheap = &heap->subheap;
subheap->user_value = heap;
subheap_set_bounds( subheap, (char *)heap + commit_size, (char *)heap + total_size );
block_size = (SIZE_T)ROUND_ADDR( subheap_size( subheap ) - subheap_overhead( subheap ), BLOCK_ALIGN - 1 );
block_init_free( first_block( subheap ), flags, subheap, block_size );
insert_free_block( heap, flags, subheap, first_block( subheap ) );
list_add_head( &heap->subheap_list, &subheap->entry );
heap_set_debug_flags( heap );
if (heap->flags & HEAP_GROWABLE)
{
SIZE_T size = (sizeof(struct bin) + sizeof(struct group *) * ARRAY_SIZE(affinity_mapping)) * BLOCK_SIZE_BIN_COUNT;
NtAllocateVirtualMemory( NtCurrentProcess(), (void *)&heap->bins,
0, &size, MEM_COMMIT, PAGE_READWRITE );
for (i = 0; heap->bins && i < BLOCK_SIZE_BIN_COUNT; ++i)
{
RtlInitializeSListHead( &heap->bins[i].groups );
/* offset affinity_group_base to interleave the bin affinity group pointers */
heap->bins[i].affinity_group_base = (struct group **)(heap->bins + BLOCK_SIZE_BIN_COUNT) + i;
}
}
/* link it into the per-process heap list */
if (process_heap)
{
RtlEnterCriticalSection( &process_heap->cs );
list_add_head( &process_heap->entry, &heap->entry );
RtlLeaveCriticalSection( &process_heap->cs );
}
else if (!addr)
{
process_heap = heap; /* assume the first heap we create is the process main heap */
list_init( &process_heap->entry );
}
return heap;
}
/***********************************************************************
* RtlDestroyHeap (NTDLL.@)
*
* Destroy a Heap created with RtlCreateHeap().
*
* PARAMS
* heap [I] Heap to destroy.
*
* RETURNS
* Success: A NULL HANDLE, if heap is NULL or it was destroyed
* Failure: The Heap handle, if heap is the process heap.
*/
HANDLE WINAPI RtlDestroyHeap( HANDLE handle )
{
SUBHEAP *subheap, *next;
ARENA_LARGE *arena, *arena_next;
struct block **pending, **tmp;
struct heap *heap;
ULONG heap_flags;
SIZE_T size;
void *addr;
TRACE( "handle %p\n", handle );
if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags )) && handle &&
(((struct heap *)handle)->flags & HEAP_VALIDATE_PARAMS) &&
NtCurrentTeb()->Peb->BeingDebugged)
{
DbgPrint( "Attempt to destroy an invalid heap\n" );
DbgBreakPoint();
}
if (!heap) return handle;
if ((pending = heap->pending_free))
{
heap->pending_free = NULL;
for (tmp = pending; *tmp && tmp != pending + MAX_FREE_PENDING; ++tmp)
{
if (!heap_free_block_lfh( heap, heap->flags, *tmp )) continue;
heap_free_block( heap, heap->flags, *tmp );
}
RtlFreeHeap( handle, 0, pending );
}
if (heap == process_heap) return handle; /* cannot delete the main process heap */
/* remove it from the per-process list */
RtlEnterCriticalSection( &process_heap->cs );
list_remove( &heap->entry );
RtlLeaveCriticalSection( &process_heap->cs );
heap->cs.DebugInfo->Spare[0] = 0;
RtlDeleteCriticalSection( &heap->cs );
LIST_FOR_EACH_ENTRY_SAFE( arena, arena_next, &heap->large_list, ARENA_LARGE, entry )
{
list_remove( &arena->entry );
size = 0;
addr = arena;
NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
}
LIST_FOR_EACH_ENTRY_SAFE( subheap, next, &heap->subheap_list, SUBHEAP, entry )
{
if (subheap == &heap->subheap) continue; /* do this one last */
valgrind_notify_free_all( subheap, heap );
list_remove( &subheap->entry );
size = 0;
addr = ROUND_ADDR( subheap, REGION_ALIGN - 1 );
NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
}
valgrind_notify_free_all( &heap->subheap, heap );
if ((addr = heap->bins))
{
size = 0;
NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
}
size = 0;
addr = heap;
NtFreeVirtualMemory( NtCurrentProcess(), &addr, &size, MEM_RELEASE );
return 0;
}
static SIZE_T heap_get_block_size( const struct heap *heap, ULONG flags, SIZE_T size )
{
static const ULONG padd_flags = HEAP_VALIDATE | HEAP_VALIDATE_ALL | HEAP_VALIDATE_PARAMS | HEAP_ADD_USER_INFO;
static const ULONG check_flags = HEAP_TAIL_CHECKING_ENABLED | HEAP_FREE_CHECKING_ENABLED | HEAP_CHECKING_ENABLED;
SIZE_T overhead, block_size;
if ((flags & check_flags)) overhead = BLOCK_ALIGN;
else overhead = sizeof(struct block);
if ((flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) overhead += BLOCK_ALIGN;
if (flags & padd_flags) overhead += BLOCK_ALIGN;
if (size < BLOCK_ALIGN) size = BLOCK_ALIGN;
block_size = ROUND_SIZE( size + overhead, BLOCK_ALIGN - 1 );
if (block_size < size) return ~0U; /* overflow */
if (block_size < HEAP_MIN_BLOCK_SIZE) block_size = HEAP_MIN_BLOCK_SIZE;
return block_size;
}
static NTSTATUS heap_allocate_block( struct heap *heap, ULONG flags, SIZE_T block_size, SIZE_T size, void **ret )
{
struct block *block, *next;
SIZE_T old_block_size;
SUBHEAP *subheap;
/* Locate a suitable free block */
if (!(block = find_free_block( heap, flags, block_size ))) return STATUS_NO_MEMORY;
/* read the free block size, changing block type or flags may alter it */
old_block_size = block_get_size( block );
subheap = block_get_subheap( heap, block );
if ((next = split_block( heap, flags, block, old_block_size, block_size )))
{
block_init_free( next, flags, subheap, old_block_size - block_size );
insert_free_block( heap, flags, subheap, next );
}
block_set_type( block, BLOCK_TYPE_USED );
block_set_flags( block, ~0, BLOCK_USER_FLAGS( flags ) );
block->tail_size = block_get_size( block ) - sizeof(*block) - size;
initialize_block( block, 0, size, flags );
mark_block_tail( block, flags );
if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
*ret = block + 1;
return STATUS_SUCCESS;
}
/* Low Fragmentation Heap frontend */
/* header for every LFH block group */
struct DECLSPEC_ALIGN(BLOCK_ALIGN) group
{
SLIST_ENTRY entry;
/* one bit for each free block and the highest bit for GROUP_FLAG_FREE */
LONG free_bits;
/* affinity of the thread which last allocated from this group */
LONG affinity;
/* first block of a group, required for alignment */
struct block first_block;
};
#define GROUP_BLOCK_COUNT (sizeof(((struct group *)0)->free_bits) * 8 - 1)
#define GROUP_FLAG_FREE (1u << GROUP_BLOCK_COUNT)
static inline UINT block_get_group_index( const struct block *block )
{
return block->base_offset;
}
static inline struct group *block_get_group( const struct block *block )
{
SIZE_T block_size = block_get_size( block );
void *first_block = (char *)block - block_get_group_index( block ) * block_size;
return CONTAINING_RECORD( first_block, struct group, first_block );
}
static inline void block_set_group( struct block *block, SIZE_T block_size, const struct group *group )
{
SIZE_T offset = (char *)block - (char *)&group->first_block;
block->base_offset = offset / block_size;
}
static inline struct block *group_get_block( struct group *group, SIZE_T block_size, UINT index )
{
char *first_block = (char *)&group->first_block;
return (struct block *)(first_block + index * block_size);
}
/* lookup a free block using the group free_bits, the current thread must own the group */
static inline struct block *group_find_free_block( struct group *group, SIZE_T block_size )
{
ULONG i, free_bits = ReadNoFence( &group->free_bits );
/* free_bits will never be 0 as the group is unlinked when it's fully used */
BitScanForward( &i, free_bits );
InterlockedAnd( &group->free_bits, ~(1 << i) );
return group_get_block( group, block_size, i );
}
/* allocate a new group block using non-LFH allocation, returns a group owned by current thread */
static struct group *group_allocate( struct heap *heap, ULONG flags, SIZE_T block_size )
{
SIZE_T i, group_size, group_block_size;
struct group *group;
NTSTATUS status;
group_size = offsetof( struct group, first_block ) + GROUP_BLOCK_COUNT * block_size;
group_block_size = heap_get_block_size( heap, flags, group_size );
heap_lock( heap, flags );
if (group_block_size >= HEAP_MIN_LARGE_BLOCK_SIZE)
status = heap_allocate_large( heap, flags & ~HEAP_ZERO_MEMORY, group_block_size, group_size, (void **)&group );
else
status = heap_allocate_block( heap, flags & ~HEAP_ZERO_MEMORY, group_block_size, group_size, (void **)&group );
heap_unlock( heap, flags );
if (status) return NULL;
block_set_flags( (struct block *)group - 1, 0, BLOCK_FLAG_LFH );
group->free_bits = ~GROUP_FLAG_FREE;
for (i = 0; i < GROUP_BLOCK_COUNT; ++i)
{
struct block *block = group_get_block( group, block_size, i );
valgrind_make_writable( block, sizeof(*block) );
block_set_type( block, BLOCK_TYPE_FREE );
block_set_flags( block, ~0, BLOCK_FLAG_FREE | BLOCK_FLAG_LFH );
block_set_group( block, block_size, group );
block_set_size( block, block_size );
mark_block_free( block + 1, (char *)block + block_size - (char *)(block + 1), flags );
}
return group;
}
/* release a fully freed group to the non-LFH subheap, group must be owned by current thread */
static NTSTATUS group_release( struct heap *heap, ULONG flags, struct bin *bin, struct group *group )
{
struct block *block = (struct block *)group - 1;
NTSTATUS status;
heap_lock( heap, flags );
block_set_flags( block, BLOCK_FLAG_LFH, 0 );
if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
status = heap_free_large( heap, flags, block );
else
status = heap_free_block( heap, flags, block );
heap_unlock( heap, flags );
return status;
}
static inline ULONG heap_current_thread_affinity(void)
{
ULONG affinity;
if (!(affinity = NtCurrentTeb()->HeapVirtualAffinity))
{
affinity = InterlockedIncrement( &next_thread_affinity );
affinity = affinity_mapping[affinity % ARRAY_SIZE(affinity_mapping)];
NtCurrentTeb()->HeapVirtualAffinity = affinity;
}
return affinity;
}
/* acquire a group from the bin, thread takes ownership of a shared group or allocates a new one */
static struct group *heap_acquire_bin_group( struct heap *heap, ULONG flags, SIZE_T block_size, struct bin *bin )
{
ULONG affinity = NtCurrentTeb()->HeapVirtualAffinity;
struct group *group;
SLIST_ENTRY *entry;
if ((group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), NULL )))
return group;
if ((entry = RtlInterlockedPopEntrySList( &bin->groups )))
return CONTAINING_RECORD( entry, struct group, entry );
return group_allocate( heap, flags, block_size );
}
/* release a thread owned and fully freed group to the bin shared group, or free its memory */
static NTSTATUS heap_release_bin_group( struct heap *heap, ULONG flags, struct bin *bin, struct group *group )
{
ULONG affinity = group->affinity;
/* using InterlockedExchangePointer here would possibly return a group that has used blocks,
* we prefer keeping our fully freed group instead for reduced memory consumption.
*/
if (!InterlockedCompareExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), group, NULL ))
return STATUS_SUCCESS;
/* try re-using the block group instead of releasing it */
if (RtlQueryDepthSList( &bin->groups ) <= ARRAY_SIZE(affinity_mapping))
{
RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
return STATUS_SUCCESS;
}
return group_release( heap, flags, bin, group );
}
static struct block *find_free_bin_block( struct heap *heap, ULONG flags, SIZE_T block_size, struct bin *bin )
{
ULONG affinity = heap_current_thread_affinity();
struct block *block;
struct group *group;
/* acquire a group, the thread will own it and no other thread can clear free bits.
* some other thread might still set the free bits if they are freeing blocks.
*/
if (!(group = heap_acquire_bin_group( heap, flags, block_size, bin ))) return NULL;
group->affinity = affinity;
block = group_find_free_block( group, block_size );
/* serialize with heap_free_block_lfh: atomically set GROUP_FLAG_FREE when the free bits are all 0. */
if (ReadNoFence( &group->free_bits ) || InterlockedCompareExchange( &group->free_bits, GROUP_FLAG_FREE, 0 ))
{
/* if GROUP_FLAG_FREE isn't set, thread is responsible for putting it back into group list. */
if ((group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), group )))
RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
}
return block;
}
static NTSTATUS heap_allocate_block_lfh( struct heap *heap, ULONG flags, SIZE_T block_size,
SIZE_T size, void **ret )
{
struct bin *bin, *last = heap->bins + BLOCK_SIZE_BIN_COUNT - 1;
struct block *block;
bin = heap->bins + BLOCK_SIZE_BIN( block_size );
if (bin == last) return STATUS_UNSUCCESSFUL;
/* paired with WriteRelease in bin_try_enable. */
if (!ReadAcquire( &bin->enabled )) return STATUS_UNSUCCESSFUL;
block_size = BLOCK_BIN_SIZE( BLOCK_SIZE_BIN( block_size ) );
if ((block = find_free_bin_block( heap, flags, block_size, bin )))
{
block_set_type( block, BLOCK_TYPE_USED );
block_set_flags( block, ~BLOCK_FLAG_LFH, BLOCK_USER_FLAGS( flags ) );
block->tail_size = block_size - sizeof(*block) - size;
initialize_block( block, 0, size, flags );
mark_block_tail( block, flags );
*ret = block + 1;
}
return block ? STATUS_SUCCESS : STATUS_NO_MEMORY;
}
static NTSTATUS heap_free_block_lfh( struct heap *heap, ULONG flags, struct block *block )
{
struct bin *bin, *last = heap->bins + BLOCK_SIZE_BIN_COUNT - 1;
SIZE_T i, block_size = block_get_size( block );
struct group *group = block_get_group( block );
NTSTATUS status = STATUS_SUCCESS;
if (!(block_get_flags( block ) & BLOCK_FLAG_LFH)) return STATUS_UNSUCCESSFUL;
bin = heap->bins + BLOCK_SIZE_BIN( block_size );
if (bin == last) return STATUS_UNSUCCESSFUL;
i = block_get_group_index( block );
valgrind_make_writable( block, sizeof(*block) );
block_set_type( block, BLOCK_TYPE_FREE );
block_set_flags( block, ~BLOCK_FLAG_LFH, BLOCK_FLAG_FREE );
mark_block_free( block + 1, (char *)block + block_size - (char *)(block + 1), flags );
/* if this was the last used block in a group and GROUP_FLAG_FREE was set */
if (InterlockedOr( &group->free_bits, 1 << i ) == ~(1 << i))
{
/* thread now owns the group, and can release it to its bin */
group->free_bits = ~GROUP_FLAG_FREE;
status = heap_release_bin_group( heap, flags, bin, group );
}
return status;
}
static void bin_try_enable( struct heap *heap, struct bin *bin )
{
ULONG alloc = ReadNoFence( &bin->count_alloc ), freed = ReadNoFence( &bin->count_freed );
SIZE_T block_size = BLOCK_BIN_SIZE( bin - heap->bins );
BOOL enable = FALSE;
if (bin == heap->bins && alloc > 0x10) enable = TRUE;
else if (bin - heap->bins < 0x30 && alloc > 0x800) enable = TRUE;
else if (bin - heap->bins < 0x30 && alloc - freed > 0x10) enable = TRUE;
else if (alloc - freed > 0x400000 / block_size) enable = TRUE;
if (!enable) return;
if (ReadNoFence( &heap->compat_info ) != HEAP_LFH)
{
ULONG info = HEAP_LFH;
RtlSetHeapInformation( heap, HeapCompatibilityInformation, &info, sizeof(info) );
}
/* paired with ReadAcquire in heap_allocate_block_lfh.
*
* The acq/rel barrier on the enabled flag is protecting compat_info
* (i.e. compat_info := LFH happens-before enabled := TRUE), so that
* a caller that observes LFH block allocation (alloc request
* succeeds without heap lock) will never observe HEAP_STD when it
* queries the heap.
*/
WriteRelease( &bin->enabled, TRUE );
}
static void heap_thread_detach_bin_groups( struct heap *heap )
{
ULONG i, affinity = NtCurrentTeb()->HeapVirtualAffinity;
if (!heap->bins) return;
for (i = 0; i < BLOCK_SIZE_BIN_COUNT; ++i)
{
struct bin *bin = heap->bins + i;
struct group *group;
if (!(group = InterlockedExchangePointer( (void *)bin_get_affinity_group( bin, affinity ), NULL ))) continue;
RtlInterlockedPushEntrySList( &bin->groups, &group->entry );
}
}
void heap_thread_detach(void)
{
struct heap *heap;
RtlEnterCriticalSection( &process_heap->cs );
LIST_FOR_EACH_ENTRY( heap, &process_heap->entry, struct heap, entry )
heap_thread_detach_bin_groups( heap );
heap_thread_detach_bin_groups( process_heap );
RtlLeaveCriticalSection( &process_heap->cs );
}
/***********************************************************************
* RtlAllocateHeap (NTDLL.@)
*/
void *WINAPI DECLSPEC_HOTPATCH RtlAllocateHeap( HANDLE handle, ULONG flags, SIZE_T size )
{
struct heap *heap;
SIZE_T block_size;
void *ptr = NULL;
ULONG heap_flags;
NTSTATUS status;
heap = unsafe_heap_from_handle( handle, flags, &heap_flags );
if ((block_size = heap_get_block_size( heap, heap_flags, size )) == ~0U)
status = STATUS_NO_MEMORY;
else if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE)
status = heap_allocate_large( heap, heap_flags, block_size, size, &ptr );
else if (heap->bins && !heap_allocate_block_lfh( heap, heap_flags, block_size, size, &ptr ))
status = STATUS_SUCCESS;
else
{
heap_lock( heap, heap_flags );
status = heap_allocate_block( heap, heap_flags, block_size, size, &ptr );
heap_unlock( heap, heap_flags );
if (!status && heap->bins)
{
SIZE_T bin = BLOCK_SIZE_BIN( block_get_size( (struct block *)ptr - 1 ) );
InterlockedIncrement( &heap->bins[bin].count_alloc );
if (!ReadNoFence( &heap->bins[bin].enabled )) bin_try_enable( heap, &heap->bins[bin] );
}
}
if (!status) valgrind_notify_alloc( ptr, size, flags & HEAP_ZERO_MEMORY );
TRACE( "handle %p, flags %#lx, size %#Ix, return %p, status %#lx.\n", handle, flags, size, ptr, status );
heap_set_status( heap, flags, status );
return ptr;
}
/***********************************************************************
* RtlFreeHeap (NTDLL.@)
*/
BOOLEAN WINAPI DECLSPEC_HOTPATCH RtlFreeHeap( HANDLE handle, ULONG flags, void *ptr )
{
struct block *block;
struct heap *heap;
ULONG heap_flags;
NTSTATUS status;
if (!ptr) return TRUE;
valgrind_notify_free( ptr );
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
status = STATUS_INVALID_PARAMETER;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
status = STATUS_INVALID_PARAMETER;
else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
status = heap_free_large( heap, heap_flags, block );
else if (!(block = heap_delay_free( heap, heap_flags, block )))
status = STATUS_SUCCESS;
else if (!heap_free_block_lfh( heap, heap_flags, block ))
status = STATUS_SUCCESS;
else
{
SIZE_T block_size = block_get_size( block ), bin = BLOCK_SIZE_BIN( block_size );
heap_lock( heap, heap_flags );
status = heap_free_block( heap, heap_flags, block );
heap_unlock( heap, heap_flags );
if (!status && heap->bins) InterlockedIncrement( &heap->bins[bin].count_freed );
}
TRACE( "handle %p, flags %#lx, ptr %p, return %u, status %#lx.\n", handle, flags, ptr, !status, status );
heap_set_status( heap, flags, status );
return !status;
}
static NTSTATUS heap_resize_large( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
SIZE_T size, SIZE_T *old_size, void **ret )
{
ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
SIZE_T old_block_size = large->block_size;
*old_size = large->data_size;
if (old_block_size < block_size) return STATUS_NO_MEMORY;
/* FIXME: we could remap zero-pages instead */
valgrind_notify_resize( block + 1, *old_size, size );
initialize_block( block, *old_size, size, flags );
large->data_size = size;
valgrind_make_noaccess( (char *)block + sizeof(*block) + large->data_size,
old_block_size - sizeof(*block) - large->data_size );
*ret = block + 1;
return STATUS_SUCCESS;
}
static NTSTATUS heap_resize_block( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
SIZE_T size, SIZE_T old_block_size, SIZE_T *old_size, void **ret )
{
SUBHEAP *subheap = block_get_subheap( heap, block );
struct block *next;
if (block_size > old_block_size)
{
/* need to grow block, make sure it's followed by large enough free block */
if (!(next = next_block( subheap, block ))) return STATUS_NO_MEMORY;
if (!(block_get_flags( next ) & BLOCK_FLAG_FREE)) return STATUS_NO_MEMORY;
if (block_size > old_block_size + block_get_size( next )) return STATUS_NO_MEMORY;
if (!subheap_commit( heap, subheap, block, block_size )) return STATUS_NO_MEMORY;
}
if ((next = next_block( subheap, block )) && (block_get_flags( next ) & BLOCK_FLAG_FREE))
{
/* merge with next block if it is free */
struct entry *entry = (struct entry *)next;
list_remove( &entry->entry );
old_block_size += block_get_size( next );
}
if ((next = split_block( heap, flags, block, old_block_size, block_size )))
{
block_init_free( next, flags, subheap, old_block_size - block_size );
insert_free_block( heap, flags, subheap, next );
}
valgrind_notify_resize( block + 1, *old_size, size );
block_set_flags( block, BLOCK_FLAG_USER_MASK & ~BLOCK_FLAG_USER_INFO, BLOCK_USER_FLAGS( flags ) );
block->tail_size = block_get_size( block ) - sizeof(*block) - size;
initialize_block( block, *old_size, size, flags );
mark_block_tail( block, flags );
if ((next = next_block( subheap, block ))) block_set_flags( next, BLOCK_FLAG_PREV_FREE, 0 );
*ret = block + 1;
return STATUS_SUCCESS;
}
static NTSTATUS heap_resize_block_lfh( struct block *block, ULONG flags, SIZE_T block_size, SIZE_T size, SIZE_T *old_size, void **ret )
{
/* as native LFH does it with different block size: refuse to resize even though we could */
if (ROUND_SIZE( *old_size, BLOCK_ALIGN - 1) != ROUND_SIZE( size, BLOCK_ALIGN - 1)) return STATUS_NO_MEMORY;
if (size >= *old_size) return STATUS_NO_MEMORY;
block_set_flags( block, BLOCK_FLAG_USER_MASK & ~BLOCK_FLAG_USER_INFO, BLOCK_USER_FLAGS( flags ) );
block->tail_size = block_size - sizeof(*block) - size;
initialize_block( block, *old_size, size, flags );
mark_block_tail( block, flags );
*ret = block + 1;
return STATUS_SUCCESS;
}
static NTSTATUS heap_resize_in_place( struct heap *heap, ULONG flags, struct block *block, SIZE_T block_size,
SIZE_T size, SIZE_T *old_size, void **ret )
{
SIZE_T old_bin, old_block_size;
NTSTATUS status;
if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
return heap_resize_large( heap, flags, block, block_size, size, old_size, ret );
old_block_size = block_get_size( block );
*old_size = old_block_size - block_get_overhead( block );
old_bin = BLOCK_SIZE_BIN( old_block_size );
if (block_size >= HEAP_MIN_LARGE_BLOCK_SIZE) return STATUS_NO_MEMORY; /* growing small block to large block */
if (block_get_flags( block ) & BLOCK_FLAG_LFH)
return heap_resize_block_lfh( block, flags, block_size, size, old_size, ret );
heap_lock( heap, flags );
status = heap_resize_block( heap, flags, block, block_size, size, old_block_size, old_size, ret );
heap_unlock( heap, flags );
if (!status && heap->bins)
{
SIZE_T new_bin = BLOCK_SIZE_BIN( block_size );
InterlockedIncrement( &heap->bins[old_bin].count_freed );
InterlockedIncrement( &heap->bins[new_bin].count_alloc );
if (!ReadNoFence( &heap->bins[new_bin].enabled )) bin_try_enable( heap, &heap->bins[new_bin] );
}
return status;
}
/***********************************************************************
* RtlReAllocateHeap (NTDLL.@)
*/
void *WINAPI RtlReAllocateHeap( HANDLE handle, ULONG flags, void *ptr, SIZE_T size )
{
SIZE_T block_size, old_size;
struct block *block;
struct heap *heap;
ULONG heap_flags;
void *ret = NULL;
NTSTATUS status;
if (!ptr) return NULL;
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
status = STATUS_INVALID_HANDLE;
else if ((block_size = heap_get_block_size( heap, heap_flags, size )) == ~0U)
status = STATUS_NO_MEMORY;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
status = STATUS_INVALID_PARAMETER;
else if ((status = heap_resize_in_place( heap, heap_flags, block, block_size, size,
&old_size, &ret )))
{
if (flags & HEAP_REALLOC_IN_PLACE_ONLY)
status = STATUS_NO_MEMORY;
else if (!(ret = RtlAllocateHeap( heap, flags, size )))
status = STATUS_NO_MEMORY;
else
{
memcpy( ret, ptr, min( size, old_size ) );
RtlFreeHeap( heap, flags, ptr );
status = STATUS_SUCCESS;
}
}
TRACE( "handle %p, flags %#lx, ptr %p, size %#Ix, return %p, status %#lx.\n", handle, flags, ptr, size, ret, status );
heap_set_status( heap, flags, status );
return ret;
}
/***********************************************************************
* RtlCompactHeap (NTDLL.@)
*
* Compact the free space in a Heap.
*
* PARAMS
* heap [I] Heap that block was allocated from
* flags [I] HEAP_ flags from "winnt.h"
*
* RETURNS
* The number of bytes compacted.
*
* NOTES
* This function is a harmless stub.
*/
ULONG WINAPI RtlCompactHeap( HANDLE handle, ULONG flags )
{
static BOOL reported;
if (!reported++) FIXME( "handle %p, flags %#lx stub!\n", handle, flags );
return 0;
}
/***********************************************************************
* RtlLockHeap (NTDLL.@)
*
* Lock a Heap.
*
* PARAMS
* heap [I] Heap to lock
*
* RETURNS
* Success: TRUE. The Heap is locked.
* Failure: FALSE, if heap is invalid.
*/
BOOLEAN WINAPI RtlLockHeap( HANDLE handle )
{
struct heap *heap;
ULONG heap_flags;
if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags ))) return FALSE;
heap_lock( heap, heap_flags );
return TRUE;
}
/***********************************************************************
* RtlUnlockHeap (NTDLL.@)
*
* Unlock a Heap.
*
* PARAMS
* heap [I] Heap to unlock
*
* RETURNS
* Success: TRUE. The Heap is unlocked.
* Failure: FALSE, if heap is invalid.
*/
BOOLEAN WINAPI RtlUnlockHeap( HANDLE handle )
{
struct heap *heap;
ULONG heap_flags;
if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags ))) return FALSE;
heap_unlock( heap, heap_flags );
return TRUE;
}
static NTSTATUS heap_size( const struct heap *heap, struct block *block, SIZE_T *size )
{
if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
{
const ARENA_LARGE *large_arena = CONTAINING_RECORD( block, ARENA_LARGE, block );
*size = large_arena->data_size;
}
else *size = block_get_size( block ) - block_get_overhead( block );
return STATUS_SUCCESS;
}
/***********************************************************************
* RtlSizeHeap (NTDLL.@)
*/
SIZE_T WINAPI RtlSizeHeap( HANDLE handle, ULONG flags, const void *ptr )
{
SIZE_T size = ~(SIZE_T)0;
struct block *block;
struct heap *heap;
ULONG heap_flags;
NTSTATUS status;
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
status = STATUS_INVALID_PARAMETER;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
status = STATUS_INVALID_PARAMETER;
else
{
heap_lock( heap, heap_flags );
status = heap_size( heap, block, &size );
heap_unlock( heap, heap_flags );
}
TRACE( "handle %p, flags %#lx, ptr %p, return %#Ix, status %#lx.\n", handle, flags, ptr, size, status );
heap_set_status( heap, flags, status );
return size;
}
/***********************************************************************
* RtlValidateHeap (NTDLL.@)
*/
BOOLEAN WINAPI RtlValidateHeap( HANDLE handle, ULONG flags, const void *ptr )
{
struct heap *heap;
ULONG heap_flags;
BOOLEAN ret;
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
ret = FALSE;
else
{
heap_lock( heap, heap_flags );
if (ptr) ret = heap_validate_ptr( heap, ptr );
else ret = heap_validate( heap );
heap_unlock( heap, heap_flags );
}
TRACE( "handle %p, flags %#lx, ptr %p, return %u.\n", handle, flags, ptr, !!ret );
return ret;
}
static NTSTATUS heap_walk_blocks( const struct heap *heap, const SUBHEAP *subheap,
const struct block *block, struct rtl_heap_entry *entry )
{
const char *base = subheap_base( subheap ), *commit_end = subheap_commit_end( subheap ), *end = base + subheap_size( subheap );
const struct block *blocks = first_block( subheap );
if (entry->lpData == commit_end) return STATUS_NO_MORE_ENTRIES;
if (entry->lpData == base) block = blocks;
else if (!(block = next_block( subheap, block )))
{
entry->lpData = (void *)commit_end;
entry->cbData = end - commit_end;
entry->cbOverhead = 0;
entry->iRegionIndex = 0;
entry->wFlags = RTL_HEAP_ENTRY_UNCOMMITTED;
return STATUS_SUCCESS;
}
if (block_get_flags( block ) & BLOCK_FLAG_FREE)
{
entry->lpData = (char *)block + block_get_overhead( block );
entry->cbData = block_get_size( block ) - block_get_overhead( block );
/* FIXME: last free block should not include uncommitted range, which also has its own overhead */
if (!contains( blocks, commit_end - 4 * BLOCK_ALIGN - (char *)blocks, block, block_get_size( block ) ))
entry->cbData = commit_end - 4 * BLOCK_ALIGN - (char *)entry->lpData;
entry->cbOverhead = 2 * BLOCK_ALIGN;
entry->iRegionIndex = 0;
entry->wFlags = 0;
}
else
{
entry->lpData = (void *)(block + 1);
entry->cbData = block_get_size( block ) - block_get_overhead( block );
entry->cbOverhead = block_get_overhead( block );
entry->iRegionIndex = 0;
entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY;
}
return STATUS_SUCCESS;
}
static NTSTATUS heap_walk( const struct heap *heap, struct rtl_heap_entry *entry )
{
const char *data = entry->lpData;
const ARENA_LARGE *large = NULL;
const struct block *block;
const struct list *next;
const SUBHEAP *subheap;
NTSTATUS status;
char *base;
if (!data || entry->wFlags & RTL_HEAP_ENTRY_REGION) block = (struct block *)data;
else if (entry->wFlags & RTL_HEAP_ENTRY_BUSY) block = (struct block *)data - 1;
else block = (struct block *)(data - sizeof(struct list)) - 1;
if (find_large_block( heap, block ))
{
large = CONTAINING_RECORD( block, ARENA_LARGE, block );
next = &large->entry;
}
else if ((subheap = find_subheap( heap, block, TRUE )))
{
if (!(status = heap_walk_blocks( heap, subheap, block, entry ))) return STATUS_SUCCESS;
else if (status != STATUS_NO_MORE_ENTRIES) return status;
next = &subheap->entry;
}
else
{
if (entry->lpData) return STATUS_INVALID_PARAMETER;
next = &heap->subheap_list;
}
if (!large && (next = list_next( &heap->subheap_list, next )))
{
subheap = LIST_ENTRY( next, SUBHEAP, entry );
base = subheap_base( subheap );
entry->lpData = base;
entry->cbData = subheap_overhead( subheap );
entry->cbOverhead = 0;
entry->iRegionIndex = 0;
entry->wFlags = RTL_HEAP_ENTRY_REGION;
entry->Region.dwCommittedSize = (char *)subheap_commit_end( subheap ) - base;
entry->Region.dwUnCommittedSize = subheap_size( subheap ) - entry->Region.dwCommittedSize;
entry->Region.lpFirstBlock = base + entry->cbData;
entry->Region.lpLastBlock = base + subheap_size( subheap );
return STATUS_SUCCESS;
}
if (!next) next = &heap->large_list;
if ((next = list_next( &heap->large_list, next )))
{
large = LIST_ENTRY( next, ARENA_LARGE, entry );
entry->lpData = (void *)(large + 1);
entry->cbData = large->data_size;
entry->cbOverhead = 0;
entry->iRegionIndex = 64;
entry->wFlags = RTL_HEAP_ENTRY_COMMITTED|RTL_HEAP_ENTRY_BLOCK|RTL_HEAP_ENTRY_BUSY;
return STATUS_SUCCESS;
}
return STATUS_NO_MORE_ENTRIES;
}
/***********************************************************************
* RtlWalkHeap (NTDLL.@)
*/
NTSTATUS WINAPI RtlWalkHeap( HANDLE handle, void *entry_ptr )
{
struct rtl_heap_entry *entry = entry_ptr;
struct heap *heap;
ULONG heap_flags;
NTSTATUS status;
if (!entry) return STATUS_INVALID_PARAMETER;
if (!(heap = unsafe_heap_from_handle( handle, 0, &heap_flags )))
status = STATUS_INVALID_HANDLE;
else
{
heap_lock( heap, heap_flags );
status = heap_walk( heap, entry );
heap_unlock( heap, heap_flags );
}
TRACE( "handle %p, entry %p %s, return %#lx\n", handle, entry,
status ? "<empty>" : debugstr_heap_entry(entry), status );
return status;
}
/***********************************************************************
* RtlGetProcessHeaps (NTDLL.@)
*
* Get the Heaps belonging to the current process.
*
* PARAMS
* count [I] size of heaps
* heaps [O] Destination array for heap HANDLE's
*
* RETURNS
* Success: The number of Heaps allocated by the process.
* Failure: 0.
*/
ULONG WINAPI RtlGetProcessHeaps( ULONG count, HANDLE *heaps )
{
ULONG total = 1; /* main heap */
struct list *ptr;
RtlEnterCriticalSection( &process_heap->cs );
LIST_FOR_EACH( ptr, &process_heap->entry ) total++;
if (total <= count)
{
*heaps++ = process_heap;
LIST_FOR_EACH( ptr, &process_heap->entry )
*heaps++ = LIST_ENTRY( ptr, struct heap, entry );
}
RtlLeaveCriticalSection( &process_heap->cs );
return total;
}
/***********************************************************************
* RtlQueryHeapInformation (NTDLL.@)
*/
NTSTATUS WINAPI RtlQueryHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class,
void *info, SIZE_T size_in, SIZE_T *size_out )
{
struct heap *heap;
ULONG flags;
TRACE( "handle %p, info_class %u, info %p, size_in %Iu, size_out %p.\n", handle, info_class, info, size_in, size_out );
switch (info_class)
{
case HeapCompatibilityInformation:
if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_ACCESS_VIOLATION;
if (size_out) *size_out = sizeof(ULONG);
if (size_in < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL;
*(ULONG *)info = ReadNoFence( &heap->compat_info );
return STATUS_SUCCESS;
default:
FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class );
return STATUS_INVALID_INFO_CLASS;
}
}
/***********************************************************************
* RtlSetHeapInformation (NTDLL.@)
*/
NTSTATUS WINAPI RtlSetHeapInformation( HANDLE handle, HEAP_INFORMATION_CLASS info_class, void *info, SIZE_T size )
{
struct heap *heap;
ULONG flags;
TRACE( "handle %p, info_class %u, info %p, size %Iu.\n", handle, info_class, info, size );
switch (info_class)
{
case HeapCompatibilityInformation:
{
ULONG compat_info;
if (size < sizeof(ULONG)) return STATUS_BUFFER_TOO_SMALL;
if (!(heap = unsafe_heap_from_handle( handle, 0, &flags ))) return STATUS_INVALID_HANDLE;
if (heap->flags & HEAP_NO_SERIALIZE) return STATUS_INVALID_PARAMETER;
compat_info = *(ULONG *)info;
if (compat_info != HEAP_STD && compat_info != HEAP_LFH)
{
FIXME( "HeapCompatibilityInformation %lu not implemented!\n", compat_info );
return STATUS_UNSUCCESSFUL;
}
if (InterlockedCompareExchange( &heap->compat_info, compat_info, HEAP_STD ) != HEAP_STD)
return STATUS_UNSUCCESSFUL;
return STATUS_SUCCESS;
}
default:
FIXME( "HEAP_INFORMATION_CLASS %u not implemented!\n", info_class );
return STATUS_SUCCESS;
}
}
/***********************************************************************
* RtlGetUserInfoHeap (NTDLL.@)
*/
BOOLEAN WINAPI RtlGetUserInfoHeap( HANDLE handle, ULONG flags, void *ptr, void **user_value, ULONG *user_flags )
{
NTSTATUS status = STATUS_SUCCESS;
struct block *block;
struct heap *heap;
ULONG heap_flags;
char *tmp;
TRACE( "handle %p, flags %#lx, ptr %p, user_value %p, user_flags %p semi-stub!\n",
handle, flags, ptr, user_value, user_flags );
*user_flags = 0;
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
status = STATUS_SUCCESS;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
{
status = STATUS_INVALID_PARAMETER;
*user_value = 0;
}
else if (!(*user_flags = HEAP_USER_FLAGS(block_get_flags( block ))))
WARN( "Block %p wasn't allocated with user info\n", ptr );
else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
{
const ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
*user_flags = *user_flags & ~HEAP_ADD_USER_INFO;
*user_value = large->user_value;
}
else
{
heap_lock( heap, heap_flags );
tmp = (char *)block + block_get_size( block ) - block->tail_size + sizeof(void *);
if ((heap_flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) tmp += BLOCK_ALIGN;
*user_flags = *user_flags & ~HEAP_ADD_USER_INFO;
*user_value = *(void **)tmp;
heap_unlock( heap, heap_flags );
}
heap_set_status( heap, flags, status );
return !status;
}
/***********************************************************************
* RtlSetUserValueHeap (NTDLL.@)
*/
BOOLEAN WINAPI RtlSetUserValueHeap( HANDLE handle, ULONG flags, void *ptr, void *user_value )
{
struct block *block;
struct heap *heap;
ULONG heap_flags;
BOOLEAN ret;
char *tmp;
TRACE( "handle %p, flags %#lx, ptr %p, user_value %p.\n", handle, flags, ptr, user_value );
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
ret = TRUE;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
ret = FALSE;
else if (!(block_get_flags( block ) & BLOCK_FLAG_USER_INFO))
ret = FALSE;
else if (block_get_flags( block ) & BLOCK_FLAG_LARGE)
{
ARENA_LARGE *large = CONTAINING_RECORD( block, ARENA_LARGE, block );
large->user_value = user_value;
ret = TRUE;
}
else
{
heap_lock( heap, heap_flags );
tmp = (char *)block + block_get_size( block ) - block->tail_size + sizeof(void *);
if ((heap_flags & HEAP_TAIL_CHECKING_ENABLED) || RUNNING_ON_VALGRIND) tmp += BLOCK_ALIGN;
*(void **)tmp = user_value;
ret = TRUE;
heap_unlock( heap, heap_flags );
}
return ret;
}
/***********************************************************************
* RtlSetUserFlagsHeap (NTDLL.@)
*/
BOOLEAN WINAPI RtlSetUserFlagsHeap( HANDLE handle, ULONG flags, void *ptr, ULONG clear, ULONG set )
{
struct block *block;
struct heap *heap;
ULONG heap_flags;
BOOLEAN ret;
TRACE( "handle %p, flags %#lx, ptr %p, clear %#lx, set %#lx.\n", handle, flags, ptr, clear, set );
if ((clear | set) & ~(0xe00))
{
SetLastError( ERROR_INVALID_PARAMETER );
return FALSE;
}
if (!(heap = unsafe_heap_from_handle( handle, flags, &heap_flags )))
ret = TRUE;
else if (!(block = unsafe_block_from_ptr( heap, heap_flags, ptr )))
ret = FALSE;
else if (!(block_get_flags( block ) & BLOCK_FLAG_USER_INFO))
ret = FALSE;
else
{
block_set_flags( block, BLOCK_USER_FLAGS( clear ), BLOCK_USER_FLAGS( set ) );
ret = TRUE;
}
return ret;
}
Reference in a new issue View git blame Copy permalink