From 39cfc5731695c46a337606da9bc86a659dbad5b3 Mon Sep 17 00:00:00 2001 From: Robin Haberkorn Date: Sun, 5 Mar 2017 18:15:05 +0100 Subject: replaced Linux-specific mallinfo()-based memory limiting with a more portable and faster hack * Works by "hooking" into malloc() and friends and counting the usable heap object sizes with malloc_usable_size(). Thus, it has no memory-overhead. * Will work at least on Linux and (Free)BSD. Other UNIXoid systems may work as well - this is tested by ./configure. * Usually faster than even the fallback implementation since the memory limit is hit earlier. * A similar approach could be tried on Windows (TODO). * A proper memory-limiting counting all malloc()s in the system can make a huge difference as this test case shows: sciteco -e '<@EU[X^E\a]"^E\a"%a>' It will allocate gigabytes before hitting the 500MB memory limit... * Fixed the UNIX-function checks on BSDs. --- TODO | 25 +++----------- configure.ac | 19 +++++----- src/memory.cpp | 107 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 45 deletions(-) diff --git a/TODO b/TODO index 2488da9..9230fed 100644 --- a/TODO +++ b/TODO @@ -249,26 +249,11 @@ Features: Macros may retrieve the code and string of the last error. Optimizations: - * The Linux-specific memory limiting using mallinfo() is - very slow (50% to 150% slower than the fallback implementation - measured in batch mode). - The fallback implementation does not come with much of a - runtime penalty. - Still I've found no faster way of measuring the process heap. - A soft resource limit would be ideal but unfortunately, - it lets malloc() return NULL and we're not in control of all - the mallocs, so glib could abort before we have a chance to - react on it. - Since the slow-down affects interactive mode as well, disabling - limiting in batch mode is merely a workaround. - Perhaps Linux should simply use the fallback limiting as well - (or support this via a configure option). - On Windows (2000), the overhead is approx. the same. - * Another Linux/glibc-specific workaround may be to hook into - all malloc(), realloc() and free() calls and count the - "usable" size of each heap object, thus avoiding mallinfo(). - Malloc hooks are deprecated, but the symbols are weak and - can be overwritten. + * The Windows-specific memory limiting using GetProcessMemoryInfo() + is very slow. Perhaps a similar approach to the generic UNIX + malloc() hooking can be implemented and memory_usage counted + with _msize() from MSVCRT. + This must be benchmarked. * Add G_UNLIKELY to all error throws. * Instead of using RTTI to implement the immediate editing command behaviours in Cmdline::process_edit_cmd() depending on the current diff --git a/configure.ac b/configure.ac index 928749b..e47ac0d 100644 --- a/configure.ac +++ b/configure.ac @@ -159,20 +159,23 @@ AC_CHECK_FUNCS([memset setlocale strchr strrchr fstat], , [ AC_MSG_ERROR([Missing libc function]) ]) -# Library functions that should exist on UNIX/Linux -# and UNIXoid systems +# Library functions that we assume exist on UNIX/Linux +# and UNIXoid systems, so that G_OS_UNIX is sufficient +# to test for them. +# FIXME: Perhaps it would be more elegant to check whether +# glib defines G_OS_UNIX||G_OS_HAIKU instead... case $host in -*-*-darwin* | *-*-linux* | *-*-cygwin* | *-*-haiku*) - AC_CHECK_FUNCS([realpath fchown dup dup2], , [ +*-*-linux* | *-*-*bsd* | *-*-darwin* | *-*-cygwin* | *-*-haiku*) + AC_CHECK_FUNCS([realpath fchown dup dup2 dlsym], , [ AC_MSG_ERROR([Missing libc function]) ]) ;; esac -# Check for optional GNU libc features. -# Will probably only be found on Linux. -AC_CHECK_HEADERS([malloc.h]) -AC_CHECK_FUNCS([malloc_trim mallinfo]) +# Check for optional libc features. +# Will probably only be found on Linux/glibc or BSD. +AC_CHECK_HEADERS([malloc.h malloc_np.h]) +AC_CHECK_FUNCS([malloc_trim malloc_usable_size]) # # Config options diff --git a/src/memory.cpp b/src/memory.cpp index 638caf9..98a2ac0 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -19,9 +19,16 @@ #include "config.h" #endif +#include #ifdef HAVE_MALLOC_H #include #endif +#ifdef HAVE_MALLOC_NP_H +#include +#endif +#ifdef HAVE_DLSYM +#include +#endif #include @@ -41,27 +48,87 @@ namespace SciTECO { MemoryLimit memlimit; -#ifdef HAVE_MALLINFO +#if defined(HAVE_DLSYM) && defined(HAVE_MALLOC_USABLE_SIZE) +/* + * This should work on most UNIXoid systems. + * + * We "hook" into the malloc-functions and count the + * "usable" size of each memory block (which may be + * more than what has been requested). + * This effectively counts all allocations by malloc(), + * g_malloc() and any C++ new() everywhere, has minimal overhead and + * is much faster than the Linux-specific mallinfo(). + */ + +static gsize memory_usage = 0; gsize MemoryLimit::get_usage(void) { - struct mallinfo info = mallinfo(); + return memory_usage; +} - /* - * NOTE: `uordblks` is an int and thus prone - * to wrap-around issues. - * - * Unfortunately, the only other machine readable - * alternative is malloc_info() which prints - * into a FILE * stream [sic!] and is unspeakably - * slow even if writing to an unbuffered fmemopen()ed - * stream. - */ - return info.uordblks; +extern "C" { + +void * +malloc(size_t size) +{ + typedef void *(*malloc_cb)(size_t); + static malloc_cb libc_malloc = NULL; + void *ret; + + if (G_UNLIKELY(!libc_malloc)) + libc_malloc = (malloc_cb)dlsym(RTLD_NEXT, "malloc"); + + ret = libc_malloc(size); + memory_usage += malloc_usable_size(ret); + + return ret; +} + +void * +realloc(void *ptr, size_t size) +{ + typedef void *(*realloc_cb)(void *, size_t); + static realloc_cb libc_realloc = NULL; + + if (G_UNLIKELY(!libc_realloc)) + libc_realloc = (realloc_cb)dlsym(RTLD_NEXT, "realloc"); + + if (ptr) + memory_usage -= malloc_usable_size(ptr); + ptr = libc_realloc(ptr, size); + memory_usage += malloc_usable_size(ptr); + + return ptr; } +void +free(void *ptr) +{ + typedef void (*free_cb)(void *); + static free_cb libc_free = NULL; + + if (G_UNLIKELY(!libc_free)) + libc_free = (free_cb)dlsym(RTLD_NEXT, "free"); + + if (ptr) + memory_usage -= malloc_usable_size(ptr); + libc_free(ptr); +} + +} /* extern "C" */ + #elif defined(G_OS_WIN32) +/* + * Uses the Windows-specific GetProcessMemoryInfo(), + * so the entire process heap is measured. + * + * FIXME: Unfortunately, this is much slower than the portable + * fallback implementation. + * We should try and benchmark a similar approach to the + * UNIX implementation above using MSVCRT-specific APIs (_minfo()). + */ gsize MemoryLimit::get_usage(void) @@ -87,8 +154,14 @@ MemoryLimit::get_usage(void) } #else +/* + * Portable fallback-implementation relying on C++11 sized allocators. + * + * Unfortunately, this will only measure the heap used by C++ objects + * in SciTECO's sources; not even Scintilla, nor all g_malloc() calls. + */ -#define USE_MEMORY_COUNTING +#define MEMORY_USAGE_FALLBACK static gsize memory_usage = 0; @@ -98,7 +171,7 @@ MemoryLimit::get_usage(void) return memory_usage; } -#endif /* !HAVE_MALLINFO && !G_OS_WIN32 */ +#endif /* (!HAVE_DLSYM || !HAVE_MALLOC_USABLE_SIZE) && !G_OS_WIN32 */ void MemoryLimit::set_limit(gsize new_limit) @@ -138,7 +211,7 @@ MemoryLimit::check(void) void * Object::operator new(size_t size) noexcept { -#ifdef USE_MEMORY_COUNTING +#ifdef MEMORY_USAGE_FALLBACK memory_usage += size; #endif @@ -168,7 +241,7 @@ Object::operator delete(void *ptr, size_t size) noexcept { g_free(ptr); -#ifdef USE_MEMORY_COUNTING +#ifdef MEMORY_USAGE_FALLBACK memory_usage -= size; #endif } -- cgit v1.2.3