/* $NetBSD: mem.c,v 1.12.2.2 2024/02/25 15:47:17 martin Exp $ */ /* * Copyright (C) Internet Systems Consortium, Inc. ("ISC") * * SPDX-License-Identifier: MPL-2.0 * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, you can obtain one at https://mozilla.org/MPL/2.0/. * * See the COPYRIGHT file distributed with this work for additional * information regarding copyright ownership. */ /*! \file */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_LIBXML2 #include #define ISC_XMLCHAR (const xmlChar *) #endif /* HAVE_LIBXML2 */ #ifdef HAVE_JSON_C #include #endif /* HAVE_JSON_C */ /* On DragonFly BSD the header does not provide jemalloc API */ #if defined(HAVE_MALLOC_NP_H) && !defined(__DragonFly__) #include #define JEMALLOC_API_SUPPORTED 1 #elif defined(HAVE_JEMALLOC) #include #define JEMALLOC_API_SUPPORTED 1 #if JEMALLOC_VERSION_MAJOR < 4 #define sdallocx(ptr, size, flags) dallocx(ptr, flags) #define MALLOCX_TCACHE_NONE (0) #endif /* JEMALLOC_VERSION_MAJOR < 4 */ #else #include "jemalloc_shim.h" #endif #include "mem_p.h" #define MCTXLOCK(m) LOCK(&m->lock) #define MCTXUNLOCK(m) UNLOCK(&m->lock) #ifndef ISC_MEM_DEBUGGING #define ISC_MEM_DEBUGGING 0 #endif /* ifndef ISC_MEM_DEBUGGING */ unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING; unsigned int isc_mem_defaultflags = ISC_MEMFLAG_DEFAULT; #define ISC_MEM_ILLEGAL_ARENA (UINT_MAX) /* * Constants. */ #define ZERO_ALLOCATION_SIZE sizeof(void *) #define ALIGNMENT 8U /*%< must be a power of 2 */ #define ALIGNMENT_SIZE sizeof(size_info) #define DEBUG_TABLE_COUNT 512U #define STATS_BUCKETS 512U #define STATS_BUCKET_SIZE 32U /* * Types. */ #if ISC_MEM_TRACKLINES typedef struct debuglink debuglink_t; struct debuglink { ISC_LINK(debuglink_t) link; const void *ptr; size_t size; const char *file; unsigned int line; }; typedef ISC_LIST(debuglink_t) debuglist_t; #define FLARG_PASS , file, line #define FLARG , const char *file, unsigned int line #else /* if ISC_MEM_TRACKLINES */ #define FLARG_PASS #define FLARG #endif /* if ISC_MEM_TRACKLINES */ typedef struct element element; struct element { element *next; }; struct stats { atomic_size_t gets; atomic_size_t totalgets; }; #define MEM_MAGIC ISC_MAGIC('M', 'e', 'm', 'C') #define VALID_CONTEXT(c) ISC_MAGIC_VALID(c, MEM_MAGIC) /* List of all active memory contexts. */ static ISC_LIST(isc_mem_t) contexts; static isc_once_t init_once = ISC_ONCE_INIT; static isc_once_t shut_once = ISC_ONCE_INIT; static isc_mutex_t contextslock; /*% * Total size of lost memory due to a bug of external library. * Locked by the global lock. */ static uint64_t totallost; struct isc_mem { unsigned int magic; unsigned int flags; unsigned int jemalloc_flags; unsigned int jemalloc_arena; isc_mutex_t lock; bool checkfree; struct stats stats[STATS_BUCKETS + 1]; isc_refcount_t references; char name[16]; atomic_size_t total; atomic_size_t inuse; atomic_size_t maxinuse; atomic_size_t malloced; atomic_size_t maxmalloced; atomic_bool hi_called; atomic_bool is_overmem; isc_mem_water_t water; void *water_arg; atomic_size_t hi_water; atomic_size_t lo_water; ISC_LIST(isc_mempool_t) pools; unsigned int poolcnt; #if ISC_MEM_TRACKLINES debuglist_t *debuglist; size_t debuglistcnt; #endif /* if ISC_MEM_TRACKLINES */ ISC_LINK(isc_mem_t) link; }; #define MEMPOOL_MAGIC ISC_MAGIC('M', 'E', 'M', 'p') #define VALID_MEMPOOL(c) ISC_MAGIC_VALID(c, MEMPOOL_MAGIC) struct isc_mempool { /* always unlocked */ unsigned int magic; isc_mem_t *mctx; /*%< our memory context */ ISC_LINK(isc_mempool_t) link; /*%< next pool in this mem context */ element *items; /*%< low water item list */ size_t size; /*%< size of each item on this pool */ size_t allocated; /*%< # of items currently given out */ size_t freecount; /*%< # of items on reserved list */ size_t freemax; /*%< # of items allowed on free list */ size_t fillcount; /*%< # of items to fetch on each fill */ /*%< Stats only. */ size_t gets; /*%< # of requests to this pool */ /*%< Debugging only. */ char name[16]; /*%< printed name in stats reports */ }; /* * Private Inline-able. */ #if !ISC_MEM_TRACKLINES #define ADD_TRACE(a, b, c, d, e) #define DELETE_TRACE(a, b, c, d, e) #define ISC_MEMFUNC_SCOPE #else /* if !ISC_MEM_TRACKLINES */ #define TRACE_OR_RECORD (ISC_MEM_DEBUGTRACE | ISC_MEM_DEBUGRECORD) #define SHOULD_TRACE_OR_RECORD(ptr) \ ((isc_mem_debugging & TRACE_OR_RECORD) != 0 && ptr != NULL) #define ADD_TRACE(a, b, c, d, e) \ if (SHOULD_TRACE_OR_RECORD(b)) { \ add_trace_entry(a, b, c, d, e); \ } #define DELETE_TRACE(a, b, c, d, e) \ if (SHOULD_TRACE_OR_RECORD(b)) { \ delete_trace_entry(a, b, c, d, e); \ } static void print_active(isc_mem_t *ctx, FILE *out); #endif /* ISC_MEM_TRACKLINES */ static size_t increment_malloced(isc_mem_t *ctx, size_t size) { size_t malloced = atomic_fetch_add_relaxed(&ctx->malloced, size) + size; size_t maxmalloced = atomic_load_relaxed(&ctx->maxmalloced); if (malloced > maxmalloced) { atomic_compare_exchange_strong(&ctx->maxmalloced, &maxmalloced, malloced); } return (malloced); } static size_t decrement_malloced(isc_mem_t *ctx, size_t size) { size_t malloced = atomic_fetch_sub_relaxed(&ctx->malloced, size) - size; return (malloced); } #if ISC_MEM_TRACKLINES /*! * mctx must not be locked. */ static void add_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size FLARG) { debuglink_t *dl = NULL; uint32_t hash; uint32_t idx; MCTXLOCK(mctx); if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "add %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { goto unlock; } #ifdef __COVERITY__ /* * Use simple conversion from pointer to hash to avoid * tainting 'ptr' due to byte swap in isc_hash_function. */ hash = (uintptr_t)ptr >> 3; #else hash = isc_hash_function(&ptr, sizeof(ptr), true); #endif idx = hash % DEBUG_TABLE_COUNT; dl = mallocx(sizeof(*dl), mctx->jemalloc_flags); INSIST(dl != NULL); increment_malloced(mctx, sizeof(*dl)); ISC_LINK_INIT(dl, link); dl->ptr = ptr; dl->size = size; dl->file = file; dl->line = line; ISC_LIST_PREPEND(mctx->debuglist[idx], dl, link); mctx->debuglistcnt++; unlock: MCTXUNLOCK(mctx); } static void delete_trace_entry(isc_mem_t *mctx, const void *ptr, size_t size, const char *file, unsigned int line) { debuglink_t *dl = NULL; uint32_t hash; uint32_t idx; MCTXLOCK(mctx); if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "del %p size %zu file %s line %u mctx %p\n", ptr, size, file, line, mctx); } if (mctx->debuglist == NULL) { goto unlock; } #ifdef __COVERITY__ /* * Use simple conversion from pointer to hash to avoid * tainting 'ptr' due to byte swap in isc_hash_function. */ hash = (uintptr_t)ptr >> 3; #else hash = isc_hash_function(&ptr, sizeof(ptr), true); #endif idx = hash % DEBUG_TABLE_COUNT; dl = ISC_LIST_HEAD(mctx->debuglist[idx]); while (dl != NULL) { if (dl->ptr == ptr) { ISC_LIST_UNLINK(mctx->debuglist[idx], dl, link); decrement_malloced(mctx, sizeof(*dl)); sdallocx(dl, sizeof(*dl), mctx->jemalloc_flags); goto unlock; } dl = ISC_LIST_NEXT(dl, link); } /* * If we get here, we didn't find the item on the list. We're * screwed. */ UNREACHABLE(); unlock: MCTXUNLOCK(mctx); } #endif /* ISC_MEM_TRACKLINES */ #define ADJUST_ZERO_ALLOCATION_SIZE(s) \ if (s == 0) { \ s = ZERO_ALLOCATION_SIZE; \ } #define MEM_ALIGN(a) ((a) ? MALLOCX_ALIGN(a) : 0) /*! * Perform a malloc, doing memory filling and overrun detection as necessary. */ static void * mem_get(isc_mem_t *ctx, size_t size, int flags) { char *ret = NULL; ADJUST_ZERO_ALLOCATION_SIZE(size); ret = mallocx(size, flags | ctx->jemalloc_flags); INSIST(ret != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { memset(ret, 0xbe, size); /* Mnemonic for "beef". */ } return (ret); } /*! * Perform a free, doing memory filling and overrun detection as necessary. */ /* coverity[+free : arg-1] */ static void mem_put(isc_mem_t *ctx, void *mem, size_t size, int flags) { ADJUST_ZERO_ALLOCATION_SIZE(size); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { memset(mem, 0xde, size); /* Mnemonic for "dead". */ } sdallocx(mem, size, flags | ctx->jemalloc_flags); } static void * mem_realloc(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size, int flags) { void *new_ptr = NULL; ADJUST_ZERO_ALLOCATION_SIZE(new_size); new_ptr = rallocx(old_ptr, new_size, flags | ctx->jemalloc_flags); INSIST(new_ptr != NULL); if ((ctx->flags & ISC_MEMFLAG_FILL) != 0) { ssize_t diff_size = new_size - old_size; void *diff_ptr = (uint8_t *)new_ptr + old_size; if (diff_size > 0) { /* Mnemonic for "beef". */ memset(diff_ptr, 0xbe, diff_size); } } return (new_ptr); } #define stats_bucket(ctx, size) \ ((size / STATS_BUCKET_SIZE) >= STATS_BUCKETS \ ? &ctx->stats[STATS_BUCKETS] \ : &ctx->stats[size / STATS_BUCKET_SIZE]) /*! * Update internal counters after a memory get. */ static void mem_getstats(isc_mem_t *ctx, size_t size) { struct stats *stats = stats_bucket(ctx, size); atomic_fetch_add_relaxed(&ctx->total, size); atomic_fetch_add_release(&ctx->inuse, size); atomic_fetch_add_relaxed(&stats->gets, 1); atomic_fetch_add_relaxed(&stats->totalgets, 1); increment_malloced(ctx, size); } /*! * Update internal counters after a memory put. */ static void mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) { struct stats *stats = stats_bucket(ctx, size); atomic_size_t s, g; UNUSED(ptr); s = atomic_fetch_sub_release(&ctx->inuse, size); INSIST(s >= size); g = atomic_fetch_sub_release(&stats->gets, 1); INSIST(g >= 1); decrement_malloced(ctx, size); } /* * Private. */ static bool mem_jemalloc_arena_create(unsigned int *pnew_arenano) { REQUIRE(pnew_arenano != NULL); #if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 unsigned int arenano = 0; size_t len = sizeof(arenano); int res = 0; res = mallctl("arenas.create", &arenano, &len, NULL, 0); if (res != 0) { return (false); } *pnew_arenano = arenano; return (true); #else *pnew_arenano = ISC_MEM_ILLEGAL_ARENA; return (true); #endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ } static bool mem_jemalloc_arena_destroy(unsigned int arenano) { #if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 int res = 0; char buf[256] = { 0 }; (void)snprintf(buf, sizeof(buf), "arena.%u.destroy", arenano); res = mallctl(buf, NULL, NULL, NULL, 0); if (res != 0) { return (false); } return (true); #else UNUSED(arenano); return (true); #endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ } static void mem_initialize(void) { isc_mutex_init(&contextslock); ISC_LIST_INIT(contexts); totallost = 0; } void isc__mem_initialize(void) { RUNTIME_CHECK(isc_once_do(&init_once, mem_initialize) == ISC_R_SUCCESS); } static void mem_shutdown(void) { isc__mem_checkdestroyed(); isc_mutex_destroy(&contextslock); } void isc__mem_shutdown(void) { RUNTIME_CHECK(isc_once_do(&shut_once, mem_shutdown) == ISC_R_SUCCESS); } static void mem_create(isc_mem_t **ctxp, unsigned int flags, unsigned int jemalloc_flags) { isc_mem_t *ctx = NULL; REQUIRE(ctxp != NULL && *ctxp == NULL); ctx = mallocx(sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline()) | jemalloc_flags); INSIST(ctx != NULL); *ctx = (isc_mem_t){ .magic = MEM_MAGIC, .flags = flags, .jemalloc_flags = jemalloc_flags, .jemalloc_arena = ISC_MEM_ILLEGAL_ARENA, .checkfree = true, }; isc_mutex_init(&ctx->lock); isc_refcount_init(&ctx->references, 1); atomic_init(&ctx->total, 0); atomic_init(&ctx->inuse, 0); atomic_init(&ctx->maxinuse, 0); atomic_init(&ctx->malloced, sizeof(*ctx)); atomic_init(&ctx->maxmalloced, sizeof(*ctx)); atomic_init(&ctx->hi_water, 0); atomic_init(&ctx->lo_water, 0); atomic_init(&ctx->hi_called, false); atomic_init(&ctx->is_overmem, false); for (size_t i = 0; i < STATS_BUCKETS + 1; i++) { atomic_init(&ctx->stats[i].gets, 0); atomic_init(&ctx->stats[i].totalgets, 0); } ISC_LIST_INIT(ctx->pools); #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGRECORD) != 0) { unsigned int i; ctx->debuglist = mallocx((DEBUG_TABLE_COUNT * sizeof(debuglist_t)), ctx->jemalloc_flags); INSIST(ctx->debuglist != NULL); for (i = 0; i < DEBUG_TABLE_COUNT; i++) { ISC_LIST_INIT(ctx->debuglist[i]); } increment_malloced(ctx, DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ LOCK(&contextslock); ISC_LIST_INITANDAPPEND(contexts, ctx, link); UNLOCK(&contextslock); *ctxp = ctx; } /* * Public. */ static void destroy(isc_mem_t *ctx) { unsigned int i; size_t malloced; unsigned int arena_no; LOCK(&contextslock); ISC_LIST_UNLINK(contexts, ctx, link); totallost += isc_mem_inuse(ctx); UNLOCK(&contextslock); ctx->magic = 0; arena_no = ctx->jemalloc_arena; INSIST(ISC_LIST_EMPTY(ctx->pools)); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { debuglink_t *dl; for (i = 0; i < DEBUG_TABLE_COUNT; i++) { for (dl = ISC_LIST_HEAD(ctx->debuglist[i]); dl != NULL; dl = ISC_LIST_HEAD(ctx->debuglist[i])) { if (ctx->checkfree && dl->ptr != NULL) { print_active(ctx, stderr); } INSIST(!ctx->checkfree || dl->ptr == NULL); ISC_LIST_UNLINK(ctx->debuglist[i], dl, link); sdallocx(dl, sizeof(*dl), ctx->jemalloc_flags); decrement_malloced(ctx, sizeof(*dl)); } } sdallocx(ctx->debuglist, (DEBUG_TABLE_COUNT * sizeof(debuglist_t)), ctx->jemalloc_flags); decrement_malloced(ctx, DEBUG_TABLE_COUNT * sizeof(debuglist_t)); } #endif /* if ISC_MEM_TRACKLINES */ if (ctx->checkfree) { for (i = 0; i <= STATS_BUCKETS; i++) { struct stats *stats = &ctx->stats[i]; size_t gets = atomic_load_acquire(&stats->gets); if (gets != 0U) { fprintf(stderr, "Failing assertion due to probable " "leaked memory in context %p (\"%s\") " "(stats[%u].gets == %zu).\n", ctx, ctx->name, i, gets); #if ISC_MEM_TRACKLINES print_active(ctx, stderr); #endif /* if ISC_MEM_TRACKLINES */ INSIST(gets == 0U); } } } isc_mutex_destroy(&ctx->lock); malloced = decrement_malloced(ctx, sizeof(*ctx)); if (ctx->checkfree) { INSIST(malloced == 0); } sdallocx(ctx, sizeof(*ctx), MALLOCX_ALIGN(isc_os_cacheline()) | ctx->jemalloc_flags); if (arena_no != ISC_MEM_ILLEGAL_ARENA) { RUNTIME_CHECK(mem_jemalloc_arena_destroy(arena_no) == true); } } void isc_mem_attach(isc_mem_t *source, isc_mem_t **targetp) { REQUIRE(VALID_CONTEXT(source)); REQUIRE(targetp != NULL && *targetp == NULL); isc_refcount_increment(&source->references); *targetp = source; } void isc__mem_detach(isc_mem_t **ctxp FLARG) { isc_mem_t *ctx = NULL; REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp)); ctx = *ctxp; *ctxp = NULL; if (isc_refcount_decrement(&ctx->references) == 1) { isc_refcount_destroy(&ctx->references); #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "destroy mctx %p file %s line %u\n", ctx, file, line); } #endif destroy(ctx); } } /* * isc_mem_putanddetach() is the equivalent of: * * mctx = NULL; * isc_mem_attach(ptr->mctx, &mctx); * isc_mem_detach(&ptr->mctx); * isc_mem_put(mctx, ptr, sizeof(*ptr); * isc_mem_detach(&mctx); */ void isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size, size_t alignment FLARG) { isc_mem_t *ctx = NULL; REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp)); REQUIRE(ptr != NULL); REQUIRE(size != 0); ctx = *ctxp; *ctxp = NULL; DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size, MEM_ALIGN(alignment)); if (isc_refcount_decrement(&ctx->references) == 1) { isc_refcount_destroy(&ctx->references); destroy(ctx); } } void isc__mem_destroy(isc_mem_t **ctxp FLARG) { isc_mem_t *ctx = NULL; /* * This routine provides legacy support for callers who use mctxs * without attaching/detaching. */ REQUIRE(ctxp != NULL && VALID_CONTEXT(*ctxp)); ctx = *ctxp; *ctxp = NULL; #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "destroy mctx %p file %s line %u\n", ctx, file, line); } if (isc_refcount_decrement(&ctx->references) > 1) { print_active(ctx, stderr); } #else /* if ISC_MEM_TRACKLINES */ isc_refcount_decrementz(&ctx->references); #endif /* if ISC_MEM_TRACKLINES */ isc_refcount_destroy(&ctx->references); destroy(ctx); *ctxp = NULL; } #define CALL_HI_WATER(ctx) \ { \ if (ctx->water != NULL && hi_water(ctx)) { \ (ctx->water)(ctx->water_arg, ISC_MEM_HIWATER); \ } \ } #define CALL_LO_WATER(ctx) \ { \ if ((ctx->water != NULL) && lo_water(ctx)) { \ (ctx->water)(ctx->water_arg, ISC_MEM_LOWATER); \ } \ } static bool hi_water(isc_mem_t *ctx) { size_t inuse; size_t maxinuse; size_t hiwater = atomic_load_relaxed(&ctx->hi_water); if (hiwater == 0) { return (false); } inuse = atomic_load_acquire(&ctx->inuse); if (inuse <= hiwater) { return (false); } maxinuse = atomic_load_acquire(&ctx->maxinuse); if (inuse > maxinuse) { (void)atomic_compare_exchange_strong(&ctx->maxinuse, &maxinuse, inuse); if ((isc_mem_debugging & ISC_MEM_DEBUGUSAGE) != 0) { fprintf(stderr, "maxinuse = %lu\n", (unsigned long)inuse); } } if (atomic_load_acquire(&ctx->hi_called)) { return (false); } /* We are over water (for the first time) */ atomic_store_release(&ctx->is_overmem, true); return (true); } static bool lo_water(isc_mem_t *ctx) { size_t inuse; size_t lowater = atomic_load_relaxed(&ctx->lo_water); if (lowater == 0) { return (false); } inuse = atomic_load_acquire(&ctx->inuse); if (inuse >= lowater) { return (false); } if (!atomic_load_acquire(&ctx->hi_called)) { return (false); } /* We are no longer overmem */ atomic_store_release(&ctx->is_overmem, false); return (true); } void * isc__mem_get(isc_mem_t *ctx, size_t size, size_t alignment FLARG) { void *ptr = NULL; REQUIRE(VALID_CONTEXT(ctx)); ptr = mem_get(ctx, size, MEM_ALIGN(alignment)); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); CALL_HI_WATER(ctx); return (ptr); } void isc__mem_put(isc_mem_t *ctx, void *ptr, size_t size, size_t alignment FLARG) { REQUIRE(VALID_CONTEXT(ctx)); DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size, MEM_ALIGN(alignment)); CALL_LO_WATER(ctx); } void isc_mem_waterack(isc_mem_t *ctx, int flag) { REQUIRE(VALID_CONTEXT(ctx)); if (flag == ISC_MEM_LOWATER) { atomic_store(&ctx->hi_called, false); } else if (flag == ISC_MEM_HIWATER) { atomic_store(&ctx->hi_called, true); } } #if ISC_MEM_TRACKLINES static void print_active(isc_mem_t *mctx, FILE *out) { if (mctx->debuglist != NULL) { debuglink_t *dl; unsigned int i; bool found; fprintf(out, "Dump of all outstanding memory " "allocations:\n"); found = false; for (i = 0; i < DEBUG_TABLE_COUNT; i++) { dl = ISC_LIST_HEAD(mctx->debuglist[i]); if (dl != NULL) { found = true; } while (dl != NULL) { if (dl->ptr != NULL) { fprintf(out, "\tptr %p size %zu " "file %s " "line %u\n", dl->ptr, dl->size, dl->file, dl->line); } dl = ISC_LIST_NEXT(dl, link); } } if (!found) { fprintf(out, "\tNone.\n"); } } } #endif /* if ISC_MEM_TRACKLINES */ /* * Print the stats[] on the stream "out" with suitable formatting. */ void isc_mem_stats(isc_mem_t *ctx, FILE *out) { isc_mempool_t *pool = NULL; REQUIRE(VALID_CONTEXT(ctx)); MCTXLOCK(ctx); for (size_t i = 0; i <= STATS_BUCKETS; i++) { size_t totalgets; size_t gets; struct stats *stats = &ctx->stats[i]; totalgets = atomic_load_acquire(&stats->totalgets); gets = atomic_load_acquire(&stats->gets); if (totalgets != 0U && gets != 0U) { fprintf(out, "%s%5zu: %11zu gets, %11zu rem", (i == STATS_BUCKETS) ? ">=" : " ", i, totalgets, gets); fputc('\n', out); } } /* * Note that since a pool can be locked now, these stats might * be somewhat off if the pool is in active use at the time the * stats are dumped. The link fields are protected by the * isc_mem_t's lock, however, so walking this list and * extracting integers from stats fields is always safe. */ pool = ISC_LIST_HEAD(ctx->pools); if (pool != NULL) { fprintf(out, "[Pool statistics]\n"); fprintf(out, "%15s %10s %10s %10s %10s %10s %10s %1s\n", "name", "size", "allocated", "freecount", "freemax", "fillcount", "gets", "L"); } while (pool != NULL) { fprintf(out, "%15s %10zu %10zu %10zu %10zu %10zu %10zu %10zu %s\n", pool->name, pool->size, (size_t)0, pool->allocated, pool->freecount, pool->freemax, pool->fillcount, pool->gets, "N"); pool = ISC_LIST_NEXT(pool, link); } #if ISC_MEM_TRACKLINES print_active(ctx, out); #endif /* if ISC_MEM_TRACKLINES */ MCTXUNLOCK(ctx); } void * isc__mem_allocate(isc_mem_t *ctx, size_t size FLARG) { void *ptr = NULL; REQUIRE(VALID_CONTEXT(ctx)); ptr = mem_get(ctx, size, 0); /* Recalculate the real allocated size */ size = sallocx(ptr, ctx->jemalloc_flags); mem_getstats(ctx, size); ADD_TRACE(ctx, ptr, size, file, line); CALL_HI_WATER(ctx); return (ptr); } void * isc__mem_reget(isc_mem_t *ctx, void *old_ptr, size_t old_size, size_t new_size, size_t alignment FLARG) { void *new_ptr = NULL; if (old_ptr == NULL) { REQUIRE(old_size == 0); new_ptr = isc__mem_get(ctx, new_size, alignment FLARG_PASS); } else if (new_size == 0) { isc__mem_put(ctx, old_ptr, old_size, alignment FLARG_PASS); } else { DELETE_TRACE(ctx, old_ptr, old_size, file, line); mem_putstats(ctx, old_ptr, old_size); new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, MEM_ALIGN(alignment)); mem_getstats(ctx, new_size); ADD_TRACE(ctx, new_ptr, new_size, file, line); /* * We want to postpone the call to water in edge case * where the realloc will exactly hit on the boundary of * the water and we would call water twice. */ CALL_LO_WATER(ctx); CALL_HI_WATER(ctx); } return (new_ptr); } void * isc__mem_reallocate(isc_mem_t *ctx, void *old_ptr, size_t new_size FLARG) { void *new_ptr = NULL; REQUIRE(VALID_CONTEXT(ctx)); if (old_ptr == NULL) { new_ptr = isc__mem_allocate(ctx, new_size FLARG_PASS); } else if (new_size == 0) { isc__mem_free(ctx, old_ptr FLARG_PASS); } else { size_t old_size = sallocx(old_ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, old_ptr, old_size, file, line); mem_putstats(ctx, old_ptr, old_size); new_ptr = mem_realloc(ctx, old_ptr, old_size, new_size, 0); /* Recalculate the real allocated size */ new_size = sallocx(new_ptr, ctx->jemalloc_flags); mem_getstats(ctx, new_size); ADD_TRACE(ctx, new_ptr, new_size, file, line); /* * We want to postpone the call to water in edge case * where the realloc will exactly hit on the boundary of * the water and we would call water twice. */ CALL_LO_WATER(ctx); CALL_HI_WATER(ctx); } return (new_ptr); } void isc__mem_free(isc_mem_t *ctx, void *ptr FLARG) { size_t size = 0; REQUIRE(VALID_CONTEXT(ctx)); size = sallocx(ptr, ctx->jemalloc_flags); DELETE_TRACE(ctx, ptr, size, file, line); mem_putstats(ctx, ptr, size); mem_put(ctx, ptr, size, 0); CALL_LO_WATER(ctx); } /* * Other useful things. */ char * isc__mem_strdup(isc_mem_t *mctx, const char *s FLARG) { size_t len; char *ns = NULL; REQUIRE(VALID_CONTEXT(mctx)); REQUIRE(s != NULL); len = strlen(s) + 1; ns = isc__mem_allocate(mctx, len FLARG_PASS); strlcpy(ns, s, len); return (ns); } char * isc__mem_strndup(isc_mem_t *mctx, const char *s, size_t size FLARG) { size_t len; char *ns = NULL; REQUIRE(VALID_CONTEXT(mctx)); REQUIRE(s != NULL); REQUIRE(size != 0); len = strlen(s) + 1; if (len > size) { len = size; } ns = isc__mem_allocate(mctx, len FLARG_PASS); strlcpy(ns, s, len); return (ns); } void isc_mem_setdestroycheck(isc_mem_t *ctx, bool flag) { REQUIRE(VALID_CONTEXT(ctx)); MCTXLOCK(ctx); ctx->checkfree = flag; MCTXUNLOCK(ctx); } size_t isc_mem_inuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_acquire(&ctx->inuse)); } size_t isc_mem_maxinuse(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_acquire(&ctx->maxinuse)); } size_t isc_mem_total(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_acquire(&ctx->total)); } size_t isc_mem_malloced(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_acquire(&ctx->malloced)); } size_t isc_mem_maxmalloced(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_acquire(&ctx->maxmalloced)); } void isc_mem_clearwater(isc_mem_t *mctx) { isc_mem_setwater(mctx, NULL, NULL, 0, 0); } void isc_mem_setwater(isc_mem_t *ctx, isc_mem_water_t water, void *water_arg, size_t hiwater, size_t lowater) { isc_mem_water_t oldwater; void *oldwater_arg; REQUIRE(VALID_CONTEXT(ctx)); REQUIRE(hiwater >= lowater); oldwater = ctx->water; oldwater_arg = ctx->water_arg; /* No water was set and new water is also NULL */ if (oldwater == NULL && water == NULL) { return; } /* The water function is being set for the first time */ if (oldwater == NULL) { REQUIRE(water != NULL && lowater > 0); INSIST(atomic_load(&ctx->hi_water) == 0); INSIST(atomic_load(&ctx->lo_water) == 0); ctx->water = water; ctx->water_arg = water_arg; atomic_store(&ctx->hi_water, hiwater); atomic_store(&ctx->lo_water, lowater); return; } REQUIRE((water == oldwater && water_arg == oldwater_arg) || (water == NULL && water_arg == NULL && hiwater == 0)); atomic_store(&ctx->hi_water, hiwater); atomic_store(&ctx->lo_water, lowater); if (atomic_load_acquire(&ctx->hi_called) && (atomic_load_acquire(&ctx->inuse) < lowater || lowater == 0U)) { (oldwater)(oldwater_arg, ISC_MEM_LOWATER); } } bool isc_mem_isovermem(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); return (atomic_load_relaxed(&ctx->is_overmem)); } void isc_mem_setname(isc_mem_t *ctx, const char *name) { REQUIRE(VALID_CONTEXT(ctx)); LOCK(&ctx->lock); strlcpy(ctx->name, name, sizeof(ctx->name)); UNLOCK(&ctx->lock); } const char * isc_mem_getname(isc_mem_t *ctx) { REQUIRE(VALID_CONTEXT(ctx)); if (ctx->name[0] == 0) { return (""); } return (ctx->name); } /* * Memory pool stuff */ void isc__mempool_create(isc_mem_t *restrict mctx, const size_t element_size, isc_mempool_t **restrict mpctxp FLARG) { isc_mempool_t *restrict mpctx = NULL; size_t size = element_size; REQUIRE(VALID_CONTEXT(mctx)); REQUIRE(size > 0U); REQUIRE(mpctxp != NULL && *mpctxp == NULL); /* * Mempools are stored as a linked list of element. */ if (size < sizeof(element)) { size = sizeof(element); } /* * Allocate space for this pool, initialize values, and if all * works well, attach to the memory context. */ mpctx = isc_mem_get(mctx, sizeof(isc_mempool_t)); *mpctx = (isc_mempool_t){ .size = size, .freemax = 1, .fillcount = 1, }; #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "create pool %p file %s line %u mctx %p\n", mpctx, file, line, mctx); } #endif /* ISC_MEM_TRACKLINES */ isc_mem_attach(mctx, &mpctx->mctx); mpctx->magic = MEMPOOL_MAGIC; *mpctxp = (isc_mempool_t *)mpctx; MCTXLOCK(mctx); ISC_LIST_INITANDAPPEND(mctx->pools, mpctx, link); mctx->poolcnt++; MCTXUNLOCK(mctx); } void isc_mempool_setname(isc_mempool_t *restrict mpctx, const char *name) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(name != NULL); strlcpy(mpctx->name, name, sizeof(mpctx->name)); } void isc__mempool_destroy(isc_mempool_t **restrict mpctxp FLARG) { isc_mempool_t *restrict mpctx = NULL; isc_mem_t *mctx = NULL; element *restrict item = NULL; REQUIRE(mpctxp != NULL); REQUIRE(VALID_MEMPOOL(*mpctxp)); mpctx = *mpctxp; *mpctxp = NULL; mctx = mpctx->mctx; #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "destroy pool %p file %s line %u mctx %p\n", mpctx, file, line, mctx); } #endif if (mpctx->allocated > 0) { UNEXPECTED_ERROR("mempool %s leaked memory", mpctx->name); } REQUIRE(mpctx->allocated == 0); /* * Return any items on the free list */ while (mpctx->items != NULL) { INSIST(mpctx->freecount > 0); mpctx->freecount--; item = mpctx->items; mpctx->items = item->next; mem_putstats(mctx, item, mpctx->size); mem_put(mctx, item, mpctx->size, 0); } /* * Remove our linked list entry from the memory context. */ MCTXLOCK(mctx); ISC_LIST_UNLINK(mctx->pools, mpctx, link); mctx->poolcnt--; MCTXUNLOCK(mctx); mpctx->magic = 0; isc_mem_putanddetach(&mpctx->mctx, mpctx, sizeof(isc_mempool_t)); } void * isc__mempool_get(isc_mempool_t *restrict mpctx FLARG) { element *restrict item = NULL; REQUIRE(VALID_MEMPOOL(mpctx)); mpctx->allocated++; if (mpctx->items == NULL) { isc_mem_t *mctx = mpctx->mctx; #if !__SANITIZE_ADDRESS__ const size_t fillcount = mpctx->fillcount; #else const size_t fillcount = 1; #endif /* * We need to dip into the well. Fill up our free list. */ for (size_t i = 0; i < fillcount; i++) { item = mem_get(mctx, mpctx->size, 0); mem_getstats(mctx, mpctx->size); item->next = mpctx->items; mpctx->items = item; mpctx->freecount++; } } item = mpctx->items; INSIST(item != NULL); mpctx->items = item->next; INSIST(mpctx->freecount > 0); mpctx->freecount--; mpctx->gets++; ADD_TRACE(mpctx->mctx, item, mpctx->size, file, line); return (item); } /* coverity[+free : arg-1] */ void isc__mempool_put(isc_mempool_t *restrict mpctx, void *mem FLARG) { element *restrict item = NULL; REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(mem != NULL); isc_mem_t *mctx = mpctx->mctx; const size_t freecount = mpctx->freecount; #if !__SANITIZE_ADDRESS__ const size_t freemax = mpctx->freemax; #else const size_t freemax = 0; #endif INSIST(mpctx->allocated > 0); mpctx->allocated--; DELETE_TRACE(mctx, mem, mpctx->size, file, line); /* * If our free list is full, return this to the mctx directly. */ if (freecount >= freemax) { mem_putstats(mctx, mem, mpctx->size); mem_put(mctx, mem, mpctx->size, 0); return; } /* * Otherwise, attach it to our free list and bump the counter. */ item = (element *)mem; item->next = mpctx->items; mpctx->items = item; mpctx->freecount++; } /* * Quotas */ void isc_mempool_setfreemax(isc_mempool_t *restrict mpctx, const unsigned int limit) { REQUIRE(VALID_MEMPOOL(mpctx)); mpctx->freemax = limit; } unsigned int isc_mempool_getfreemax(isc_mempool_t *restrict mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); return (mpctx->freemax); } unsigned int isc_mempool_getfreecount(isc_mempool_t *restrict mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); return (mpctx->freecount); } unsigned int isc_mempool_getallocated(isc_mempool_t *restrict mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); return (mpctx->allocated); } void isc_mempool_setfillcount(isc_mempool_t *restrict mpctx, unsigned int const limit) { REQUIRE(VALID_MEMPOOL(mpctx)); REQUIRE(limit > 0); mpctx->fillcount = limit; } unsigned int isc_mempool_getfillcount(isc_mempool_t *restrict mpctx) { REQUIRE(VALID_MEMPOOL(mpctx)); return (mpctx->fillcount); } /* * Requires contextslock to be held by caller. */ #if ISC_MEM_TRACKLINES static void print_contexts(FILE *file) { isc_mem_t *ctx; for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL; ctx = ISC_LIST_NEXT(ctx, link)) { fprintf(file, "context: %p (%s): %" PRIuFAST32 " references\n", ctx, ctx->name[0] == 0 ? "" : ctx->name, isc_refcount_current(&ctx->references)); print_active(ctx, file); } fflush(file); } #endif static atomic_uintptr_t checkdestroyed = 0; void isc_mem_checkdestroyed(FILE *file) { atomic_store_release(&checkdestroyed, (uintptr_t)file); } void isc__mem_checkdestroyed(void) { FILE *file = (FILE *)atomic_load_acquire(&checkdestroyed); if (file == NULL) { return; } LOCK(&contextslock); if (!ISC_LIST_EMPTY(contexts)) { #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & TRACE_OR_RECORD) != 0) { print_contexts(file); } #endif /* if ISC_MEM_TRACKLINES */ UNREACHABLE(); } UNLOCK(&contextslock); } unsigned int isc_mem_references(isc_mem_t *ctx) { return (isc_refcount_current(&ctx->references)); } typedef struct summarystat { uint64_t total; uint64_t inuse; uint64_t malloced; uint64_t contextsize; } summarystat_t; #ifdef HAVE_LIBXML2 #define TRY0(a) \ do { \ xmlrc = (a); \ if (xmlrc < 0) \ goto error; \ } while (0) static int xml_renderctx(isc_mem_t *ctx, summarystat_t *summary, xmlTextWriterPtr writer) { REQUIRE(VALID_CONTEXT(ctx)); int xmlrc; MCTXLOCK(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "context")); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "id")); TRY0(xmlTextWriterWriteFormatString(writer, "%p", ctx)); TRY0(xmlTextWriterEndElement(writer)); /* id */ if (ctx->name[0] != 0) { TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "name")); TRY0(xmlTextWriterWriteFormatString(writer, "%s", ctx->name)); TRY0(xmlTextWriterEndElement(writer)); /* name */ } summary->contextsize += sizeof(*ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * sizeof(debuglist_t) + ctx->debuglistcnt * sizeof(debuglink_t); } #endif /* if ISC_MEM_TRACKLINES */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "references")); TRY0(xmlTextWriterWriteFormatString( writer, "%" PRIuFAST32, isc_refcount_current(&ctx->references))); TRY0(xmlTextWriterEndElement(writer)); /* references */ summary->total += isc_mem_total(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "total")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", (uint64_t)isc_mem_total(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* total */ summary->inuse += isc_mem_inuse(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "inuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", (uint64_t)isc_mem_inuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* inuse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxinuse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", (uint64_t)isc_mem_maxinuse(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxinuse */ summary->malloced += isc_mem_malloced(ctx); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", (uint64_t)isc_mem_malloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* malloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxmalloced")); TRY0(xmlTextWriterWriteFormatString( writer, "%" PRIu64 "", (uint64_t)isc_mem_maxmalloced(ctx))); TRY0(xmlTextWriterEndElement(writer)); /* maxmalloced */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "pools")); TRY0(xmlTextWriterWriteFormatString(writer, "%u", ctx->poolcnt)); TRY0(xmlTextWriterEndElement(writer)); /* pools */ summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "hiwater")); TRY0(xmlTextWriterWriteFormatString( writer, "%" PRIu64 "", (uint64_t)atomic_load_relaxed(&ctx->hi_water))); TRY0(xmlTextWriterEndElement(writer)); /* hiwater */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "lowater")); TRY0(xmlTextWriterWriteFormatString( writer, "%" PRIu64 "", (uint64_t)atomic_load_relaxed(&ctx->lo_water))); TRY0(xmlTextWriterEndElement(writer)); /* lowater */ TRY0(xmlTextWriterEndElement(writer)); /* context */ error: MCTXUNLOCK(ctx); return (xmlrc); } int isc_mem_renderxml(void *writer0) { isc_mem_t *ctx; summarystat_t summary = { 0 }; uint64_t lost; int xmlrc; xmlTextWriterPtr writer = (xmlTextWriterPtr)writer0; TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "contexts")); LOCK(&contextslock); lost = totallost; for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL; ctx = ISC_LIST_NEXT(ctx, link)) { xmlrc = xml_renderctx(ctx, &summary, writer); if (xmlrc < 0) { UNLOCK(&contextslock); goto error; } } UNLOCK(&contextslock); TRY0(xmlTextWriterEndElement(writer)); /* contexts */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "summary")); TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "TotalUse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", summary.total)); TRY0(xmlTextWriterEndElement(writer)); /* TotalUse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "InUse")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", summary.inuse)); TRY0(xmlTextWriterEndElement(writer)); /* InUse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Malloced")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", summary.malloced)); TRY0(xmlTextWriterEndElement(writer)); /* InUse */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "ContextSize")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", summary.contextsize)); TRY0(xmlTextWriterEndElement(writer)); /* ContextSize */ TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Lost")); TRY0(xmlTextWriterWriteFormatString(writer, "%" PRIu64 "", lost)); TRY0(xmlTextWriterEndElement(writer)); /* Lost */ TRY0(xmlTextWriterEndElement(writer)); /* summary */ error: return (xmlrc); } #endif /* HAVE_LIBXML2 */ #ifdef HAVE_JSON_C #define CHECKMEM(m) RUNTIME_CHECK(m != NULL) static isc_result_t json_renderctx(isc_mem_t *ctx, summarystat_t *summary, json_object *array) { REQUIRE(VALID_CONTEXT(ctx)); REQUIRE(summary != NULL); REQUIRE(array != NULL); json_object *ctxobj, *obj; char buf[1024]; MCTXLOCK(ctx); summary->contextsize += sizeof(*ctx); summary->total += isc_mem_total(ctx); summary->inuse += isc_mem_inuse(ctx); summary->malloced += isc_mem_malloced(ctx); #if ISC_MEM_TRACKLINES if (ctx->debuglist != NULL) { summary->contextsize += DEBUG_TABLE_COUNT * sizeof(debuglist_t) + ctx->debuglistcnt * sizeof(debuglink_t); } #endif /* if ISC_MEM_TRACKLINES */ ctxobj = json_object_new_object(); CHECKMEM(ctxobj); snprintf(buf, sizeof(buf), "%p", ctx); obj = json_object_new_string(buf); CHECKMEM(obj); json_object_object_add(ctxobj, "id", obj); if (ctx->name[0] != 0) { obj = json_object_new_string(ctx->name); CHECKMEM(obj); json_object_object_add(ctxobj, "name", obj); } obj = json_object_new_int64(isc_refcount_current(&ctx->references)); CHECKMEM(obj); json_object_object_add(ctxobj, "references", obj); obj = json_object_new_int64(isc_mem_total(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "total", obj); obj = json_object_new_int64(isc_mem_inuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "inuse", obj); obj = json_object_new_int64(isc_mem_maxinuse(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxinuse", obj); obj = json_object_new_int64(isc_mem_malloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "malloced", obj); obj = json_object_new_int64(isc_mem_maxmalloced(ctx)); CHECKMEM(obj); json_object_object_add(ctxobj, "maxmalloced", obj); obj = json_object_new_int64(ctx->poolcnt); CHECKMEM(obj); json_object_object_add(ctxobj, "pools", obj); summary->contextsize += ctx->poolcnt * sizeof(isc_mempool_t); obj = json_object_new_int64(atomic_load_relaxed(&ctx->hi_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "hiwater", obj); obj = json_object_new_int64(atomic_load_relaxed(&ctx->lo_water)); CHECKMEM(obj); json_object_object_add(ctxobj, "lowater", obj); MCTXUNLOCK(ctx); json_object_array_add(array, ctxobj); return (ISC_R_SUCCESS); } isc_result_t isc_mem_renderjson(void *memobj0) { isc_result_t result = ISC_R_SUCCESS; isc_mem_t *ctx; summarystat_t summary = { 0 }; uint64_t lost; json_object *ctxarray, *obj; json_object *memobj = (json_object *)memobj0; ctxarray = json_object_new_array(); CHECKMEM(ctxarray); LOCK(&contextslock); lost = totallost; for (ctx = ISC_LIST_HEAD(contexts); ctx != NULL; ctx = ISC_LIST_NEXT(ctx, link)) { result = json_renderctx(ctx, &summary, ctxarray); if (result != ISC_R_SUCCESS) { UNLOCK(&contextslock); goto error; } } UNLOCK(&contextslock); obj = json_object_new_int64(summary.total); CHECKMEM(obj); json_object_object_add(memobj, "TotalUse", obj); obj = json_object_new_int64(summary.inuse); CHECKMEM(obj); json_object_object_add(memobj, "InUse", obj); obj = json_object_new_int64(summary.malloced); CHECKMEM(obj); json_object_object_add(memobj, "Malloced", obj); obj = json_object_new_int64(summary.contextsize); CHECKMEM(obj); json_object_object_add(memobj, "ContextSize", obj); obj = json_object_new_int64(lost); CHECKMEM(obj); json_object_object_add(memobj, "Lost", obj); json_object_object_add(memobj, "contexts", ctxarray); return (ISC_R_SUCCESS); error: if (ctxarray != NULL) { json_object_put(ctxarray); } return (result); } #endif /* HAVE_JSON_C */ void isc__mem_create(isc_mem_t **mctxp FLARG) { mem_create(mctxp, isc_mem_defaultflags, 0); #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "create mctx %p file %s line %u\n", *mctxp, file, line); } #endif /* ISC_MEM_TRACKLINES */ } void isc__mem_create_arena(isc_mem_t **mctxp FLARG) { unsigned int arena_no = ISC_MEM_ILLEGAL_ARENA; RUNTIME_CHECK(mem_jemalloc_arena_create(&arena_no)); /* * We use MALLOCX_TCACHE_NONE to bypass the tcache and route * allocations directly to the arena. That is a recommendation * from jemalloc developers: * * https://github.com/jemalloc/jemalloc/issues/2483#issuecomment-1698173849 */ mem_create(mctxp, isc_mem_defaultflags, arena_no == ISC_MEM_ILLEGAL_ARENA ? 0 : MALLOCX_ARENA(arena_no) | MALLOCX_TCACHE_NONE); (*mctxp)->jemalloc_arena = arena_no; #if ISC_MEM_TRACKLINES if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0) { fprintf(stderr, "create mctx %p file %s line %u for jemalloc arena " "%u\n", *mctxp, file, line, arena_no); } #endif /* ISC_MEM_TRACKLINES */ } #if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 static bool jemalloc_set_ssize_value(const char *valname, ssize_t newval) { int ret; ret = mallctl(valname, NULL, NULL, &newval, sizeof(newval)); return (ret == 0); } #endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ static isc_result_t mem_set_arena_ssize_value(isc_mem_t *mctx, const char *arena_valname, const ssize_t newval) { REQUIRE(VALID_CONTEXT(mctx)); #if defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 bool ret; char buf[256] = { 0 }; if (mctx->jemalloc_arena == ISC_MEM_ILLEGAL_ARENA) { return (ISC_R_UNEXPECTED); } (void)snprintf(buf, sizeof(buf), "arena.%u.%s", mctx->jemalloc_arena, arena_valname); ret = jemalloc_set_ssize_value(buf, newval); if (!ret) { return (ISC_R_FAILURE); } return (ISC_R_SUCCESS); #else UNUSED(arena_valname); UNUSED(newval); return (ISC_R_NOTIMPLEMENTED); #endif /* defined(JEMALLOC_API_SUPPORTED) && JEMALLOC_VERSION_MAJOR >= 4 */ } isc_result_t isc_mem_arena_set_muzzy_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) { return (mem_set_arena_ssize_value(mctx, "muzzy_decay_ms", decay_ms)); } isc_result_t isc_mem_arena_set_dirty_decay_ms(isc_mem_t *mctx, const ssize_t decay_ms) { return (mem_set_arena_ssize_value(mctx, "dirty_decay_ms", decay_ms)); } void isc__mem_printactive(isc_mem_t *ctx, FILE *file) { #if ISC_MEM_TRACKLINES REQUIRE(VALID_CONTEXT(ctx)); REQUIRE(file != NULL); print_active(ctx, file); #else /* if ISC_MEM_TRACKLINES */ UNUSED(ctx); UNUSED(file); #endif /* if ISC_MEM_TRACKLINES */ } void * isc__mem_alloc_noctx(size_t size) { return mallocx(size, 0); } void isc__mem_free_noctx(void *ptr, size_t size) { ADJUST_ZERO_ALLOCATION_SIZE(size); sdallocx(ptr, size, 0); }