LCOV - app.info - memory/jemalloc/jemalloc.c

LCOV - code coverage report
| Current view: | directory - memory/jemalloc - jemalloc.c (source / functions) |  | Found | Hit | Coverage |
| Test: | app.info | Lines: | 1540 | 948 | 61.6 %
| Date: | 2012-04-07 | Functions: | 139 | 79 | 56.8 %

           1                 : /* -*- Mode: C; tab-width: 8; c-basic-offset: 8; indent-tabs-mode: t -*- */
           2                 : /* vim:set softtabstop=8 shiftwidth=8 noet: */
           3                 : /*-
           4                 :  * Copyright (C) 2006-2008 Jason Evans <jasone@FreeBSD.org>.
           5                 :  * All rights reserved.
           6                 :  *
           7                 :  * Redistribution and use in source and binary forms, with or without
           8                 :  * modification, are permitted provided that the following conditions
           9                 :  * are met:
          10                 :  * 1. Redistributions of source code must retain the above copyright
          11                 :  *    notice(s), this list of conditions and the following disclaimer as
          12                 :  *    the first lines of this file unmodified other than the possible
          13                 :  *    addition of one or more copyright notices.
          14                 :  * 2. Redistributions in binary form must reproduce the above copyright
          15                 :  *    notice(s), this list of conditions and the following disclaimer in
          16                 :  *    the documentation and/or other materials provided with the
          17                 :  *    distribution.
          18                 :  *
          19                 :  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
          20                 :  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
          21                 :  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
          22                 :  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
          23                 :  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
          24                 :  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
          25                 :  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
          26                 :  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
          27                 :  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
          28                 :  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
          29                 :  * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
          30                 :  *
          31                 :  *******************************************************************************
          32                 :  *
          33                 :  * This allocator implementation is designed to provide scalable performance
          34                 :  * for multi-threaded programs on multi-processor systems.  The following
          35                 :  * features are included for this purpose:
          36                 :  *
          37                 :  *   + Multiple arenas are used if there are multiple CPUs, which reduces lock
          38                 :  *     contention and cache sloshing.
          39                 :  *
          40                 :  *   + Cache line sharing between arenas is avoided for internal data
          41                 :  *     structures.
          42                 :  *
          43                 :  *   + Memory is managed in chunks and runs (chunks can be split into runs),
          44                 :  *     rather than as individual pages.  This provides a constant-time
          45                 :  *     mechanism for associating allocations with particular arenas.
          46                 :  *
          47                 :  * Allocation requests are rounded up to the nearest size class, and no record
          48                 :  * of the original request size is maintained.  Allocations are broken into
          49                 :  * categories according to size class.  Assuming runtime defaults, 4 kB pages
          50                 :  * and a 16 byte quantum on a 32-bit system, the size classes in each category
          51                 :  * are as follows:
          52                 :  *
          53                 :  *   |=====================================|
          54                 :  *   | Category | Subcategory    |    Size |
          55                 :  *   |=====================================|
          56                 :  *   | Small    | Tiny           |       2 |
          57                 :  *   |          |                |       4 |
          58                 :  *   |          |                |       8 |
          59                 :  *   |          |----------------+---------|
          60                 :  *   |          | Quantum-spaced |      16 |
          61                 :  *   |          |                |      32 |
          62                 :  *   |          |                |      48 |
          63                 :  *   |          |                |     ... |
          64                 :  *   |          |                |     480 |
          65                 :  *   |          |                |     496 |
          66                 :  *   |          |                |     512 |
          67                 :  *   |          |----------------+---------|
          68                 :  *   |          | Sub-page       |    1 kB |
          69                 :  *   |          |                |    2 kB |
          70                 :  *   |=====================================|
          71                 :  *   | Large                     |    4 kB |
          72                 :  *   |                           |    8 kB |
          73                 :  *   |                           |   12 kB |
          74                 :  *   |                           |     ... |
          75                 :  *   |                           | 1012 kB |
          76                 :  *   |                           | 1016 kB |
          77                 :  *   |                           | 1020 kB |
          78                 :  *   |=====================================|
          79                 :  *   | Huge                      |    1 MB |
          80                 :  *   |                           |    2 MB |
          81                 :  *   |                           |    3 MB |
          82                 :  *   |                           |     ... |
          83                 :  *   |=====================================|
          84                 :  *
          85                 :  * NOTE: Due to Mozilla bug 691003, we cannot reserve less than one word for an
          86                 :  * allocation on Linux or Mac.  So on 32-bit *nix, the smallest bucket size is
          87                 :  * 4 bytes, and on 64-bit, the smallest bucket size is 8 bytes.
          88                 :  *
          89                 :  * A different mechanism is used for each category:
          90                 :  *
          91                 :  *   Small : Each size class is segregated into its own set of runs.  Each run
          92                 :  *           maintains a bitmap of which regions are free/allocated.
          93                 :  *
          94                 :  *   Large : Each allocation is backed by a dedicated run.  Metadata are stored
          95                 :  *           in the associated arena chunk header maps.
          96                 :  *
          97                 :  *   Huge : Each allocation is backed by a dedicated contiguous set of chunks.
          98                 :  *          Metadata are stored in a separate red-black tree.
          99                 :  *
         100                 :  *******************************************************************************
         101                 :  */
         102                 :
         103                 : #ifdef MOZ_MEMORY_ANDROID
         104                 : #define NO_TLS
         105                 : #define _pthread_self() pthread_self()
         106                 : #endif
         107                 :
         108                 : /*
         109                 :  * On Linux, we use madvise(MADV_DONTNEED) to release memory back to the
         110                 :  * operating system.  If we release 1MB of live pages with MADV_DONTNEED, our
         111                 :  * RSS will decrease by 1MB (almost) immediately.
         112                 :  *
         113                 :  * On Mac, we use madvise(MADV_FREE).  Unlike MADV_DONTNEED on Linux, MADV_FREE
         114                 :  * on Mac doesn't cause the OS to release the specified pages immediately; the
         115                 :  * OS keeps them in our process until the machine comes under memory pressure.
         116                 :  *
         117                 :  * It's therefore difficult to measure the process's RSS on Mac, since, in the
         118                 :  * absence of memory pressure, the contribution from the heap to RSS will not
         119                 :  * decrease due to our madvise calls.
         120                 :  *
         121                 :  * We therefore define MALLOC_DOUBLE_PURGE on Mac.  This causes jemalloc to
         122                 :  * track which pages have been MADV_FREE'd.  You can then call
         123                 :  * jemalloc_purge_freed_pages(), which will force the OS to release those
         124                 :  * MADV_FREE'd pages, making the process's RSS reflect its true memory usage.
         125                 :  *
         126                 :  * The jemalloc_purge_freed_pages definition in jemalloc.h needs to be
         127                 :  * adjusted if MALLOC_DOUBLE_PURGE is ever enabled on Linux.
         128                 :  */
         129                 : #ifdef MOZ_MEMORY_DARWIN
         130                 : #define MALLOC_DOUBLE_PURGE
         131                 : #endif
         132                 :
         133                 : /*
         134                 :  * MALLOC_PRODUCTION disables assertions and statistics gathering.  It also
         135                 :  * defaults the A and J runtime options to off.  These settings are appropriate
         136                 :  * for production systems.
         137                 :  */
         138                 : #ifndef MOZ_MEMORY_DEBUG
         139                 : #  define       MALLOC_PRODUCTION
         140                 : #endif
         141                 :
         142                 : /*
         143                 :  * Use only one arena by default.  Mozilla does not currently make extensive
         144                 :  * use of concurrent allocation, so the increased fragmentation associated with
         145                 :  * multiple arenas is not warranted.
         146                 :  */
         147                 : #define MOZ_MEMORY_NARENAS_DEFAULT_ONE
         148                 :
         149                 : /*
         150                 :  * MALLOC_STATS enables statistics calculation, and is required for
         151                 :  * jemalloc_stats().
         152                 :  */
         153                 : #define MALLOC_STATS
         154                 :
         155                 : #ifndef MALLOC_PRODUCTION
         156                 :    /*
         157                 :     * MALLOC_DEBUG enables assertions and other sanity checks, and disables
         158                 :     * inline functions.
         159                 :     */
         160                 : #  define MALLOC_DEBUG
         161                 :
         162                 :    /* Memory filling (junk/zero). */
         163                 : #  define MALLOC_FILL
         164                 :
         165                 :    /* Allocation tracing. */
         166                 : #  ifndef MOZ_MEMORY_WINDOWS
         167                 : #    define MALLOC_UTRACE
         168                 : #  endif
         169                 :
         170                 :    /* Support optional abort() on OOM. */
         171                 : #  define MALLOC_XMALLOC
         172                 :
         173                 :    /* Support SYSV semantics. */
         174                 : #  define MALLOC_SYSV
         175                 : #endif
         176                 :
         177                 : /*
         178                 :  * MALLOC_VALIDATE causes malloc_usable_size() to perform some pointer
         179                 :  * validation.  There are many possible errors that validation does not even
         180                 :  * attempt to detect.
         181                 :  */
         182                 : #define MALLOC_VALIDATE
         183                 :
         184                 : /* Embed no-op macros that support memory allocation tracking via valgrind. */
         185                 : #ifdef MOZ_VALGRIND
         186                 : #  define MALLOC_VALGRIND
         187                 : #endif
         188                 : #ifdef MALLOC_VALGRIND
         189                 : #  include <valgrind/valgrind.h>
         190                 : #else
         191                 : #  define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
         192                 : #  define VALGRIND_FREELIKE_BLOCK(addr, rzB)
         193                 : #endif
         194                 :
         195                 : /*
         196                 :  * MALLOC_BALANCE enables monitoring of arena lock contention and dynamically
         197                 :  * re-balances arena load if exponentially averaged contention exceeds a
         198                 :  * certain threshold.
         199                 :  */
         200                 : /* #define      MALLOC_BALANCE */
         201                 :
         202                 : /*
         203                 :  * MALLOC_PAGEFILE causes all mmap()ed memory to be backed by temporary
         204                 :  * files, so that if a chunk is mapped, it is guaranteed to be swappable.
         205                 :  * This avoids asynchronous OOM failures that are due to VM over-commit.
         206                 :  */
         207                 : /* #define MALLOC_PAGEFILE */
         208                 :
         209                 : #ifdef MALLOC_PAGEFILE
         210                 : /* Write size when initializing a page file. */
         211                 : #  define MALLOC_PAGEFILE_WRITE_SIZE 512
         212                 : #endif
         213                 :
         214                 : #if defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
         215                 : #define _GNU_SOURCE /* For mremap(2). */
         216                 : #define issetugid() 0
         217                 : #if 0 /* Enable in order to test decommit code on Linux. */
         218                 : #  define MALLOC_DECOMMIT
         219                 : #endif
         220                 : #endif
         221                 :
         222                 : #include <sys/types.h>
         223                 :
         224                 : #include <errno.h>
         225                 : #include <stdlib.h>
         226                 : #include <limits.h>
         227                 : #include <stdarg.h>
         228                 : #include <stdio.h>
         229                 : #include <string.h>
         230                 :
         231                 : #ifdef MOZ_MEMORY_WINDOWS
         232                 :
         233                 : /* Some defines from the CRT internal headers that we need here. */
         234                 : #define _CRT_SPINCOUNT 5000
         235                 : #define __crtInitCritSecAndSpinCount InitializeCriticalSectionAndSpinCount
         236                 : #include <io.h>
         237                 : #include <windows.h>
         238                 :
         239                 : #pragma warning( disable: 4267 4996 4146 )
         240                 :
         241                 : #define bool BOOL
         242                 : #define false FALSE
         243                 : #define true TRUE
         244                 : #define inline __inline
         245                 : #define SIZE_T_MAX SIZE_MAX
         246                 : #define STDERR_FILENO 2
         247                 : #define PATH_MAX MAX_PATH
         248                 : #define vsnprintf _vsnprintf
         249                 :
         250                 : #ifndef NO_TLS
         251                 : static unsigned long tlsIndex = 0xffffffff;
         252                 : #endif
         253                 :
         254                 : #define __thread
         255                 : #define _pthread_self() __threadid()
         256                 : #define issetugid() 0
         257                 :
         258                 : /* use MSVC intrinsics */
         259                 : #pragma intrinsic(_BitScanForward)
         260                 : static __forceinline int
         261                 : ffs(int x)
         262                 : {
         263                 :         unsigned long i;
         264                 :
         265                 :         if (_BitScanForward(&i, x) != 0)
         266                 :                 return (i + 1);
         267                 :
         268                 :         return (0);
         269                 : }
         270                 :
         271                 : /* Implement getenv without using malloc */
         272                 : static char mozillaMallocOptionsBuf[64];
         273                 :
         274                 : #define getenv xgetenv
         275                 : static char *
         276                 : getenv(const char *name)
         277                 : {
         278                 :
         279                 :         if (GetEnvironmentVariableA(name, (LPSTR)&mozillaMallocOptionsBuf,
         280                 :                     sizeof(mozillaMallocOptionsBuf)) > 0)
         281                 :                 return (mozillaMallocOptionsBuf);
         282                 :
         283                 :         return (NULL);
         284                 : }
         285                 :
         286                 : typedef unsigned char uint8_t;
         287                 : typedef unsigned uint32_t;
         288                 : typedef unsigned long long uint64_t;
         289                 : typedef unsigned long long uintmax_t;
         290                 : #if defined(MOZ_MEMORY_SIZEOF_PTR_2POW) && (MOZ_MEMORY_SIZEOF_PTR_2POW == 3)
         291                 : typedef long long ssize_t;
         292                 : #else
         293                 : typedef long ssize_t;
         294                 : #endif
         295                 :
         296                 : #define MALLOC_DECOMMIT
         297                 : #endif
         298                 :
         299                 : #ifndef MOZ_MEMORY_WINDOWS
         300                 : #ifndef MOZ_MEMORY_SOLARIS
         301                 : #include <sys/cdefs.h>
         302                 : #endif
         303                 : #ifndef __DECONST
         304                 : #  define __DECONST(type, var)  ((type)(uintptr_t)(const void *)(var))
         305                 : #endif
         306                 : #ifndef MOZ_MEMORY
         307                 : __FBSDID("$FreeBSD: head/lib/libc/stdlib/malloc.c 180599 2008-07-18 19:35:44Z jasone $");
         308                 : #include "libc_private.h"
         309                 : #ifdef MALLOC_DEBUG
         310                 : #  define _LOCK_DEBUG
         311                 : #endif
         312                 : #include "spinlock.h"
         313                 : #include "namespace.h"
         314                 : #endif
         315                 : #include <sys/mman.h>
         316                 : #ifndef MADV_FREE
         317                 : #  define MADV_FREE     MADV_DONTNEED
         318                 : #endif
         319                 : #ifndef MAP_NOSYNC
         320                 : #  define MAP_NOSYNC    0
         321                 : #endif
         322                 : #include <sys/param.h>
         323                 : #ifndef MOZ_MEMORY
         324                 : #include <sys/stddef.h>
         325                 : #endif
         326                 : #include <sys/time.h>
         327                 : #include <sys/types.h>
         328                 : #if !defined(MOZ_MEMORY_SOLARIS) && !defined(MOZ_MEMORY_ANDROID)
         329                 : #include <sys/sysctl.h>
         330                 : #endif
         331                 : #include <sys/uio.h>
         332                 : #ifndef MOZ_MEMORY
         333                 : #include <sys/ktrace.h> /* Must come after several other sys/ includes. */
         334                 :
         335                 : #include <machine/atomic.h>
         336                 : #include <machine/cpufunc.h>
         337                 : #include <machine/vmparam.h>
         338                 : #endif
         339                 :
         340                 : #include <errno.h>
         341                 : #include <limits.h>
         342                 : #ifndef SIZE_T_MAX
         343                 : #  define SIZE_T_MAX    SIZE_MAX
         344                 : #endif
         345                 : #include <pthread.h>
         346                 : #ifdef MOZ_MEMORY_DARWIN
         347                 : #define _pthread_self pthread_self
         348                 : #define _pthread_mutex_init pthread_mutex_init
         349                 : #define _pthread_mutex_trylock pthread_mutex_trylock
         350                 : #define _pthread_mutex_lock pthread_mutex_lock
         351                 : #define _pthread_mutex_unlock pthread_mutex_unlock
         352                 : #endif
         353                 : #include <sched.h>
         354                 : #include <stdarg.h>
         355                 : #include <stdio.h>
         356                 : #include <stdbool.h>
         357                 : #include <stdint.h>
         358                 : #include <stdlib.h>
         359                 : #include <string.h>
         360                 : #ifndef MOZ_MEMORY_DARWIN
         361                 : #include <strings.h>
         362                 : #endif
         363                 : #include <unistd.h>
         364                 :
         365                 : #ifdef MOZ_MEMORY_DARWIN
         366                 : #include <libkern/OSAtomic.h>
         367                 : #include <mach/mach_error.h>
         368                 : #include <mach/mach_init.h>
         369                 : #include <mach/vm_map.h>
         370                 : #include <malloc/malloc.h>
         371                 : #endif
         372                 :
         373                 : #ifndef MOZ_MEMORY
         374                 : #include "un-namespace.h"
         375                 : #endif
         376                 :
         377                 : #endif
         378                 :
         379                 : #include "jemalloc_types.h"
         380                 : #include "linkedlist.h"
         381                 :
         382                 : /* Some tools, such as /dev/dsp wrappers, LD_PRELOAD libraries that
         383                 :  * happen to override mmap() and call dlsym() from their overridden
         384                 :  * mmap(). The problem is that dlsym() calls malloc(), and this ends
         385                 :  * up in a dead lock in jemalloc.
         386                 :  * On these systems, we prefer to directly use the system call.
         387                 :  * We do that for Linux systems and kfreebsd with GNU userland.
         388                 :  * Note sanity checks are not done (alignment of offset, ...) because
         389                 :  * the uses of mmap are pretty limited, in jemalloc.
         390                 :  *
         391                 :  * On Alpha, glibc has a bug that prevents syscall() to work for system
         392                 :  * calls with 6 arguments
         393                 :  */
         394                 : #if (defined(MOZ_MEMORY_LINUX) && !defined(__alpha__)) || \
         395                 :     (defined(MOZ_MEMORY_BSD) && defined(__GLIBC__))
         396                 : #include <sys/syscall.h>
         397                 : #if defined(SYS_mmap) || defined(SYS_mmap2)
         398                 : static inline
         399           64774 : void *_mmap(void *addr, size_t length, int prot, int flags,
         400                 :             int fd, off_t offset)
         401                 : {
         402                 : /* S390 only passes one argument to the mmap system call, which is a
         403                 :  * pointer to a structure containing the arguments */
         404                 : #ifdef __s390__
         405                 :         struct {
         406                 :                 void *addr;
         407                 :                 size_t length;
         408                 :                 int prot;
         409                 :                 int flags;
         410                 :                 int fd;
         411                 :                 off_t offset;
         412                 :         } args = { addr, length, prot, flags, fd, offset };
         413                 :         return (void *) syscall(SYS_mmap, &args);
         414                 : #else
         415                 : #ifdef SYS_mmap2
         416           64774 :         return (void *) syscall(SYS_mmap2, addr, length, prot, flags,
         417                 :                                fd, offset >> 12);
         418                 : #else
         419                 :         return (void *) syscall(SYS_mmap, addr, length, prot, flags,
         420                 :                                fd, offset);
         421                 : #endif
         422                 : #endif
         423                 : }
         424                 : #define mmap _mmap
         425                 : #define munmap(a, l) syscall(SYS_munmap, a, l)
         426                 : #endif
         427                 : #endif
         428                 :
         429                 : #ifdef MOZ_MEMORY_DARWIN
         430                 : static const bool isthreaded = true;
         431                 : #endif
         432                 :
         433                 : #if defined(MOZ_MEMORY_SOLARIS) && defined(MAP_ALIGN) && !defined(JEMALLOC_NEVER_USES_MAP_ALIGN)
         434                 : #define JEMALLOC_USES_MAP_ALIGN  /* Required on Solaris 10. Might improve performance elsewhere. */
         435                 : #endif
         436                 :
         437                 : #define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var))
         438                 :
         439                 : #ifdef MOZ_MEMORY_WINDOWS
         440                 :    /* MSVC++ does not support C99 variable-length arrays. */
         441                 : #  define RB_NO_C99_VARARRAYS
         442                 : #endif
         443                 : #include "rb.h"
         444                 :
         445                 : #ifdef MALLOC_DEBUG
         446                 :    /* Disable inlining to make debugging easier. */
         447                 : #ifdef inline
         448                 : #undef inline
         449                 : #endif
         450                 :
         451                 : #  define inline
         452                 : #endif
         453                 :
         454                 : /* Size of stack-allocated buffer passed to strerror_r(). */
         455                 : #define STRERROR_BUF            64
         456                 :
         457                 : /* Minimum alignment of non-tiny allocations is 2^QUANTUM_2POW_MIN bytes. */
         458                 : #  define QUANTUM_2POW_MIN      4
         459                 : #ifdef MOZ_MEMORY_SIZEOF_PTR_2POW
         460                 : #  define SIZEOF_PTR_2POW               MOZ_MEMORY_SIZEOF_PTR_2POW
         461                 : #else
         462                 : #  define SIZEOF_PTR_2POW       2
         463                 : #endif
         464                 : #define PIC
         465                 : #ifndef MOZ_MEMORY_DARWIN
         466                 : static const bool isthreaded = true;
         467                 : #else
         468                 : #  define NO_TLS
         469                 : #endif
         470                 : #if 0
         471                 : #ifdef __i386__
         472                 : #  define QUANTUM_2POW_MIN      4
         473                 : #  define SIZEOF_PTR_2POW       2
         474                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
         475                 : #endif
         476                 : #ifdef __ia64__
         477                 : #  define QUANTUM_2POW_MIN      4
         478                 : #  define SIZEOF_PTR_2POW       3
         479                 : #endif
         480                 : #ifdef __alpha__
         481                 : #  define QUANTUM_2POW_MIN      4
         482                 : #  define SIZEOF_PTR_2POW       3
         483                 : #  define NO_TLS
         484                 : #endif
         485                 : #ifdef __sparc64__
         486                 : #  define QUANTUM_2POW_MIN      4
         487                 : #  define SIZEOF_PTR_2POW       3
         488                 : #  define NO_TLS
         489                 : #endif
         490                 : #ifdef __amd64__
         491                 : #  define QUANTUM_2POW_MIN      4
         492                 : #  define SIZEOF_PTR_2POW       3
         493                 : #  define CPU_SPINWAIT          __asm__ volatile("pause")
         494                 : #endif
         495                 : #ifdef __arm__
         496                 : #  define QUANTUM_2POW_MIN      3
         497                 : #  define SIZEOF_PTR_2POW       2
         498                 : #  define NO_TLS
         499                 : #endif
         500                 : #ifdef __mips__
         501                 : #  define QUANTUM_2POW_MIN      3
         502                 : #  define SIZEOF_PTR_2POW       2
         503                 : #  define NO_TLS
         504                 : #endif
         505                 : #ifdef __powerpc__
         506                 : #  define QUANTUM_2POW_MIN      4
         507                 : #  define SIZEOF_PTR_2POW       2
         508                 : #endif
         509                 : #endif
         510                 :
         511                 : #define SIZEOF_PTR              (1U << SIZEOF_PTR_2POW)
         512                 :
         513                 : /* sizeof(int) == (1U << SIZEOF_INT_2POW). */
         514                 : #ifndef SIZEOF_INT_2POW
         515                 : #  define SIZEOF_INT_2POW       2
         516                 : #endif
         517                 :
         518                 : /* We can't use TLS in non-PIC programs, since TLS relies on loader magic. */
         519                 : #if (!defined(PIC) && !defined(NO_TLS))
         520                 : #  define NO_TLS
         521                 : #endif
         522                 :
         523                 : #ifdef NO_TLS
         524                 :    /* MALLOC_BALANCE requires TLS. */
         525                 : #  ifdef MALLOC_BALANCE
         526                 : #    undef MALLOC_BALANCE
         527                 : #  endif
         528                 : #endif
         529                 :
         530                 : /*
         531                 :  * Size and alignment of memory chunks that are allocated by the OS's virtual
         532                 :  * memory system.
         533                 :  */
         534                 : #define CHUNK_2POW_DEFAULT      20
         535                 : /* Maximum number of dirty pages per arena. */
         536                 : #define DIRTY_MAX_DEFAULT       (1U << 10)
         537                 :
         538                 : /*
         539                 :  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing,
         540                 :  * so over-estimates are okay (up to a point), but under-estimates will
         541                 :  * negatively affect performance.
         542                 :  */
         543                 : #define CACHELINE_2POW          6
         544                 : #define CACHELINE               ((size_t)(1U << CACHELINE_2POW))
         545                 :
         546                 : /*
         547                 :  * Smallest size class to support.  On Linux and Mac, even malloc(1) must
         548                 :  * reserve a word's worth of memory (see Mozilla bug 691003).
         549                 :  */
         550                 : #ifdef MOZ_MEMORY_WINDOWS
         551                 : #define TINY_MIN_2POW           1
         552                 : #else
         553                 : #define TINY_MIN_2POW           (sizeof(void*) == 8 ? 3 : 2)
         554                 : #endif
         555                 :
         556                 : /*
         557                 :  * Maximum size class that is a multiple of the quantum, but not (necessarily)
         558                 :  * a power of 2.  Above this size, allocations are rounded up to the nearest
         559                 :  * power of 2.
         560                 :  */
         561                 : #define SMALL_MAX_2POW_DEFAULT  9
         562                 : #define SMALL_MAX_DEFAULT       (1U << SMALL_MAX_2POW_DEFAULT)
         563                 :
         564                 : /*
         565                 :  * RUN_MAX_OVRHD indicates maximum desired run header overhead.  Runs are sized
         566                 :  * as small as possible such that this setting is still honored, without
         567                 :  * violating other constraints.  The goal is to make runs as small as possible
         568                 :  * without exceeding a per run external fragmentation threshold.
         569                 :  *
         570                 :  * We use binary fixed point math for overhead computations, where the binary
         571                 :  * point is implicitly RUN_BFP bits to the left.
         572                 :  *
         573                 :  * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
         574                 :  * honored for some/all object sizes, since there is one bit of header overhead
         575                 :  * per object (plus a constant).  This constraint is relaxed (ignored) for runs
         576                 :  * that are so small that the per-region overhead is greater than:
         577                 :  *
         578                 :  *   (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
         579                 :  */
         580                 : #define RUN_BFP                 12
         581                 : /*                                    \/   Implicit binary fixed point. */
         582                 : #define RUN_MAX_OVRHD           0x0000003dU
         583                 : #define RUN_MAX_OVRHD_RELAX     0x00001800U
         584                 :
         585                 : /* Put a cap on small object run size.  This overrides RUN_MAX_OVRHD. */
         586                 : #define RUN_MAX_SMALL_2POW      15
         587                 : #define RUN_MAX_SMALL           (1U << RUN_MAX_SMALL_2POW)
         588                 :
         589                 : /*
         590                 :  * Hyper-threaded CPUs may need a special instruction inside spin loops in
         591                 :  * order to yield to another virtual CPU.  If no such instruction is defined
         592                 :  * above, make CPU_SPINWAIT a no-op.
         593                 :  */
         594                 : #ifndef CPU_SPINWAIT
         595                 : #  define CPU_SPINWAIT
         596                 : #endif
         597                 :
         598                 : /*
         599                 :  * Adaptive spinning must eventually switch to blocking, in order to avoid the
         600                 :  * potential for priority inversion deadlock.  Backing off past a certain point
         601                 :  * can actually waste time.
         602                 :  */
         603                 : #define SPIN_LIMIT_2POW         11
         604                 :
         605                 : /*
         606                 :  * Conversion from spinning to blocking is expensive; we use (1U <<
         607                 :  * BLOCK_COST_2POW) to estimate how many more times costly blocking is than
         608                 :  * worst-case spinning.
         609                 :  */
         610                 : #define BLOCK_COST_2POW         4
         611                 :
         612                 : #ifdef MALLOC_BALANCE
         613                 :    /*
         614                 :     * We use an exponential moving average to track recent lock contention,
         615                 :     * where the size of the history window is N, and alpha=2/(N+1).
         616                 :     *
         617                 :     * Due to integer math rounding, very small values here can cause
         618                 :     * substantial degradation in accuracy, thus making the moving average decay
         619                 :     * faster than it would with precise calculation.
         620                 :     */
         621                 : #  define BALANCE_ALPHA_INV_2POW        9
         622                 :
         623                 :    /*
         624                 :     * Threshold value for the exponential moving contention average at which to
         625                 :     * re-assign a thread.
         626                 :     */
         627                 : #  define BALANCE_THRESHOLD_DEFAULT     (1U << (SPIN_LIMIT_2POW-4))
         628                 : #endif
         629                 :
         630                 : /******************************************************************************/
         631                 :
         632                 : /* MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive. */
         633                 : #if defined(MALLOC_DECOMMIT) && defined(MALLOC_DOUBLE_PURGE)
         634                 : #error MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are mutually exclusive.
         635                 : #endif
         636                 :
         637                 : /*
         638                 :  * Mutexes based on spinlocks.  We can't use normal pthread spinlocks in all
         639                 :  * places, because they require malloc()ed memory, which causes bootstrapping
         640                 :  * issues in some cases.
         641                 :  */
         642                 : #if defined(MOZ_MEMORY_WINDOWS)
         643                 : #define malloc_mutex_t CRITICAL_SECTION
         644                 : #define malloc_spinlock_t CRITICAL_SECTION
         645                 : #elif defined(MOZ_MEMORY_DARWIN)
         646                 : typedef struct {
         647                 :         OSSpinLock      lock;
         648                 : } malloc_mutex_t;
         649                 : typedef struct {
         650                 :         OSSpinLock      lock;
         651                 : } malloc_spinlock_t;
         652                 : #elif defined(MOZ_MEMORY)
         653                 : typedef pthread_mutex_t malloc_mutex_t;
         654                 : typedef pthread_mutex_t malloc_spinlock_t;
         655                 : #else
         656                 : /* XXX these should #ifdef these for freebsd (and linux?) only */
         657                 : typedef struct {
         658                 :         spinlock_t      lock;
         659                 : } malloc_mutex_t;
         660                 : typedef malloc_spinlock_t malloc_mutex_t;
         661                 : #endif
         662                 :
         663                 : /* Set to true once the allocator has been initialized. */
         664                 : static bool malloc_initialized = false;
         665                 :
         666                 : #if defined(MOZ_MEMORY_WINDOWS)
         667                 : /* No init lock for Windows. */
         668                 : #elif defined(MOZ_MEMORY_DARWIN)
         669                 : static malloc_mutex_t init_lock = {OS_SPINLOCK_INIT};
         670                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
         671                 : static malloc_mutex_t init_lock = PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP;
         672                 : #elif defined(MOZ_MEMORY)
         673                 : static malloc_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
         674                 : #else
         675                 : static malloc_mutex_t init_lock = {_SPINLOCK_INITIALIZER};
         676                 : #endif
         677                 :
         678                 : /******************************************************************************/
         679                 : /*
         680                 :  * Statistics data structures.
         681                 :  */
         682                 :
         683                 : #ifdef MALLOC_STATS
         684                 :
         685                 : typedef struct malloc_bin_stats_s malloc_bin_stats_t;
         686                 : struct malloc_bin_stats_s {
         687                 :         /*
         688                 :          * Number of allocation requests that corresponded to the size of this
         689                 :          * bin.
         690                 :          */
         691                 :         uint64_t        nrequests;
         692                 :
         693                 :         /* Total number of runs created for this bin's size class. */
         694                 :         uint64_t        nruns;
         695                 :
         696                 :         /*
         697                 :          * Total number of runs reused by extracting them from the runs tree for
         698                 :          * this bin's size class.
         699                 :          */
         700                 :         uint64_t        reruns;
         701                 :
         702                 :         /* High-water mark for this bin. */
         703                 :         unsigned long   highruns;
         704                 :
         705                 :         /* Current number of runs in this bin. */
         706                 :         unsigned long   curruns;
         707                 : };
         708                 :
         709                 : typedef struct arena_stats_s arena_stats_t;
         710                 : struct arena_stats_s {
         711                 :         /* Number of bytes currently mapped. */
         712                 :         size_t          mapped;
         713                 :
         714                 :         /*
         715                 :          * Total number of purge sweeps, total number of madvise calls made,
         716                 :          * and total pages purged in order to keep dirty unused memory under
         717                 :          * control.
         718                 :          */
         719                 :         uint64_t        npurge;
         720                 :         uint64_t        nmadvise;
         721                 :         uint64_t        purged;
         722                 : #ifdef MALLOC_DECOMMIT
         723                 :         /*
         724                 :          * Total number of decommit/commit operations, and total number of
         725                 :          * pages decommitted.
         726                 :          */
         727                 :         uint64_t        ndecommit;
         728                 :         uint64_t        ncommit;
         729                 :         uint64_t        decommitted;
         730                 : #endif
         731                 :
         732                 :         /* Current number of committed pages. */
         733                 :         size_t          committed;
         734                 :
         735                 :         /* Per-size-category statistics. */
         736                 :         size_t          allocated_small;
         737                 :         uint64_t        nmalloc_small;
         738                 :         uint64_t        ndalloc_small;
         739                 :
         740                 :         size_t          allocated_large;
         741                 :         uint64_t        nmalloc_large;
         742                 :         uint64_t        ndalloc_large;
         743                 :
         744                 : #ifdef MALLOC_BALANCE
         745                 :         /* Number of times this arena reassigned a thread due to contention. */
         746                 :         uint64_t        nbalance;
         747                 : #endif
         748                 : };
         749                 :
         750                 : typedef struct chunk_stats_s chunk_stats_t;
         751                 : struct chunk_stats_s {
         752                 :         /* Number of chunks that were allocated. */
         753                 :         uint64_t        nchunks;
         754                 :
         755                 :         /* High-water mark for number of chunks allocated. */
         756                 :         unsigned long   highchunks;
         757                 :
         758                 :         /*
         759                 :          * Current number of chunks allocated.  This value isn't maintained for
         760                 :          * any other purpose, so keep track of it in order to be able to set
         761                 :          * highchunks.
         762                 :          */
         763                 :         unsigned long   curchunks;
         764                 : };
         765                 :
         766                 : #endif /* #ifdef MALLOC_STATS */
         767                 :
         768                 : /******************************************************************************/
         769                 : /*
         770                 :  * Extent data structures.
         771                 :  */
         772                 :
         773                 : /* Tree of extents. */
         774                 : typedef struct extent_node_s extent_node_t;
         775                 : struct extent_node_s {
         776                 :         /* Linkage for the size/address-ordered tree. */
         777                 :         rb_node(extent_node_t) link_szad;
         778                 :
         779                 :         /* Linkage for the address-ordered tree. */
         780                 :         rb_node(extent_node_t) link_ad;
         781                 :
         782                 :         /* Pointer to the extent that this tree node is responsible for. */
         783                 :         void    *addr;
         784                 :
         785                 :         /* Total region size. */
         786                 :         size_t  size;
         787                 : };
         788                 : typedef rb_tree(extent_node_t) extent_tree_t;
         789                 :
         790                 : /******************************************************************************/
         791                 : /*
         792                 :  * Radix tree data structures.
         793                 :  */
         794                 :
         795                 : #ifdef MALLOC_VALIDATE
         796                 :    /*
         797                 :     * Size of each radix tree node (must be a power of 2).  This impacts tree
         798                 :     * depth.
         799                 :     */
         800                 : #  if (SIZEOF_PTR == 4)
         801                 : #    define MALLOC_RTREE_NODESIZE (1U << 14)
         802                 : #  else
         803                 : #    define MALLOC_RTREE_NODESIZE CACHELINE
         804                 : #  endif
         805                 :
         806                 : typedef struct malloc_rtree_s malloc_rtree_t;
         807                 : struct malloc_rtree_s {
         808                 :         malloc_spinlock_t       lock;
         809                 :         void                    **root;
         810                 :         unsigned                height;
         811                 :         unsigned                level2bits[1]; /* Dynamically sized. */
         812                 : };
         813                 : #endif
         814                 :
         815                 : /******************************************************************************/
         816                 : /*
         817                 :  * Arena data structures.
         818                 :  */
         819                 :
         820                 : typedef struct arena_s arena_t;
         821                 : typedef struct arena_bin_s arena_bin_t;
         822                 :
         823                 : /* Each element of the chunk map corresponds to one page within the chunk. */
         824                 : typedef struct arena_chunk_map_s arena_chunk_map_t;
         825                 : struct arena_chunk_map_s {
         826                 :         /*
         827                 :          * Linkage for run trees.  There are two disjoint uses:
         828                 :          *
         829                 :          * 1) arena_t's runs_avail tree.
         830                 :          * 2) arena_run_t conceptually uses this linkage for in-use non-full
         831                 :          *    runs, rather than directly embedding linkage.
         832                 :          */
         833                 :         rb_node(arena_chunk_map_t)      link;
         834                 :
         835                 :         /*
         836                 :          * Run address (or size) and various flags are stored together.  The bit
         837                 :          * layout looks like (assuming 32-bit system):
         838                 :          *
         839                 :          *   ???????? ???????? ????---- -mckdzla
         840                 :          *
         841                 :          * ? : Unallocated: Run address for first/last pages, unset for internal
         842                 :          *                  pages.
         843                 :          *     Small: Run address.
         844                 :          *     Large: Run size for first page, unset for trailing pages.
         845                 :          * - : Unused.
         846                 :          * m : MADV_FREE/MADV_DONTNEED'ed?
         847                 :          * c : decommitted?
         848                 :          * k : key?
         849                 :          * d : dirty?
         850                 :          * z : zeroed?
         851                 :          * l : large?
         852                 :          * a : allocated?
         853                 :          *
         854                 :          * Following are example bit patterns for the three types of runs.
         855                 :          *
         856                 :          * r : run address
         857                 :          * s : run size
         858                 :          * x : don't care
         859                 :          * - : 0
         860                 :          * [cdzla] : bit set
         861                 :          *
         862                 :          *   Unallocated:
         863                 :          *     ssssssss ssssssss ssss---- --c-----
         864                 :          *     xxxxxxxx xxxxxxxx xxxx---- ----d---
         865                 :          *     ssssssss ssssssss ssss---- -----z--
         866                 :          *
         867                 :          *   Small:
         868                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
         869                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
         870                 :          *     rrrrrrrr rrrrrrrr rrrr---- -------a
         871                 :          *
         872                 :          *   Large:
         873                 :          *     ssssssss ssssssss ssss---- ------la
         874                 :          *     -------- -------- -------- ------la
         875                 :          *     -------- -------- -------- ------la
         876                 :          */
         877                 :         size_t                          bits;
         878                 :
         879                 : /* Note that CHUNK_MAP_DECOMMITTED's meaning varies depending on whether
         880                 :  * MALLOC_DECOMMIT and MALLOC_DOUBLE_PURGE are defined.
         881                 :  *
         882                 :  * If MALLOC_DECOMMIT is defined, a page which is CHUNK_MAP_DECOMMITTED must be
         883                 :  * re-committed with pages_commit() before it may be touched.  If
         884                 :  * MALLOC_DECOMMIT is defined, MALLOC_DOUBLE_PURGE may not be defined.
         885                 :  *
         886                 :  * If neither MALLOC_DECOMMIT nor MALLOC_DOUBLE_PURGE is defined, pages which
         887                 :  * are madvised (with either MADV_DONTNEED or MADV_FREE) are marked with
         888                 :  * CHUNK_MAP_MADVISED.
         889                 :  *
         890                 :  * Otherwise, if MALLOC_DECOMMIT is not defined and MALLOC_DOUBLE_PURGE is
         891                 :  * defined, then a page which is madvised is marked as CHUNK_MAP_MADVISED.
         892                 :  * When it's finally freed with jemalloc_purge_freed_pages, the page is marked
         893                 :  * as CHUNK_MAP_DECOMMITTED.
         894                 :  */
         895                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
         896                 : #define CHUNK_MAP_MADVISED      ((size_t)0x40U)
         897                 : #define CHUNK_MAP_DECOMMITTED   ((size_t)0x20U)
         898                 : #define CHUNK_MAP_MADVISED_OR_DECOMMITTED (CHUNK_MAP_MADVISED | CHUNK_MAP_DECOMMITTED)
         899                 : #endif
         900                 : #define CHUNK_MAP_KEY           ((size_t)0x10U)
         901                 : #define CHUNK_MAP_DIRTY         ((size_t)0x08U)
         902                 : #define CHUNK_MAP_ZEROED        ((size_t)0x04U)
         903                 : #define CHUNK_MAP_LARGE         ((size_t)0x02U)
         904                 : #define CHUNK_MAP_ALLOCATED     ((size_t)0x01U)
         905                 : };
         906                 : typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
         907                 : typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
         908                 :
         909                 : /* Arena chunk header. */
         910                 : typedef struct arena_chunk_s arena_chunk_t;
         911                 : struct arena_chunk_s {
         912                 :         /* Arena that owns the chunk. */
         913                 :         arena_t         *arena;
         914                 :
         915                 :         /* Linkage for the arena's chunks_dirty tree. */
         916                 :         rb_node(arena_chunk_t) link_dirty;
         917                 :
         918                 : #ifdef MALLOC_DOUBLE_PURGE
         919                 :         /* If we're double-purging, we maintain a linked list of chunks which
         920                 :          * have pages which have been madvise(MADV_FREE)'d but not explicitly
         921                 :          * purged.
         922                 :          *
         923                 :          * We're currently lazy and don't remove a chunk from this list when
         924                 :          * all its madvised pages are recommitted. */
         925                 :         LinkedList      chunks_madvised_elem;
         926                 : #endif
         927                 :
         928                 :         /* Number of dirty pages. */
         929                 :         size_t          ndirty;
         930                 :
         931                 :         /* Map of pages within chunk that keeps track of free/large/small. */
         932                 :         arena_chunk_map_t map[1]; /* Dynamically sized. */
         933                 : };
         934                 : typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
         935                 :
         936                 : typedef struct arena_run_s arena_run_t;
         937                 : struct arena_run_s {
         938                 : #ifdef MALLOC_DEBUG
         939                 :         uint32_t        magic;
         940                 : #  define ARENA_RUN_MAGIC 0x384adf93
         941                 : #endif
         942                 :
         943                 :         /* Bin this run is associated with. */
         944                 :         arena_bin_t     *bin;
         945                 :
         946                 :         /* Index of first element that might have a free region. */
         947                 :         unsigned        regs_minelm;
         948                 :
         949                 :         /* Number of free regions in run. */
         950                 :         unsigned        nfree;
         951                 :
         952                 :         /* Bitmask of in-use regions (0: in use, 1: free). */
         953                 :         unsigned        regs_mask[1]; /* Dynamically sized. */
         954                 : };
         955                 :
         956                 : struct arena_bin_s {
         957                 :         /*
         958                 :          * Current run being used to service allocations of this bin's size
         959                 :          * class.
         960                 :          */
         961                 :         arena_run_t     *runcur;
         962                 :
         963                 :         /*
         964                 :          * Tree of non-full runs.  This tree is used when looking for an
         965                 :          * existing run when runcur is no longer usable.  We choose the
         966                 :          * non-full run that is lowest in memory; this policy tends to keep
         967                 :          * objects packed well, and it can also help reduce the number of
         968                 :          * almost-empty chunks.
         969                 :          */
         970                 :         arena_run_tree_t runs;
         971                 :
         972                 :         /* Size of regions in a run for this bin's size class. */
         973                 :         size_t          reg_size;
         974                 :
         975                 :         /* Total size of a run for this bin's size class. */
         976                 :         size_t          run_size;
         977                 :
         978                 :         /* Total number of regions in a run for this bin's size class. */
         979                 :         uint32_t        nregs;
         980                 :
         981                 :         /* Number of elements in a run's regs_mask for this bin's size class. */
         982                 :         uint32_t        regs_mask_nelms;
         983                 :
         984                 :         /* Offset of first region in a run for this bin's size class. */
         985                 :         uint32_t        reg0_offset;
         986                 :
         987                 : #ifdef MALLOC_STATS
         988                 :         /* Bin statistics. */
         989                 :         malloc_bin_stats_t stats;
         990                 : #endif
         991                 : };
         992                 :
         993                 : struct arena_s {
         994                 : #ifdef MALLOC_DEBUG
         995                 :         uint32_t                magic;
         996                 : #  define ARENA_MAGIC 0x947d3d24
         997                 : #endif
         998                 :
         999                 :         /* All operations on this arena require that lock be locked. */
        1000                 : #ifdef MOZ_MEMORY
        1001                 :         malloc_spinlock_t       lock;
        1002                 : #else
        1003                 :         pthread_mutex_t         lock;
        1004                 : #endif
        1005                 :
        1006                 : #ifdef MALLOC_STATS
        1007                 :         arena_stats_t           stats;
        1008                 : #endif
        1009                 :
        1010                 :         /* Tree of dirty-page-containing chunks this arena manages. */
        1011                 :         arena_chunk_tree_t      chunks_dirty;
        1012                 :
        1013                 : #ifdef MALLOC_DOUBLE_PURGE
        1014                 :         /* Head of a linked list of MADV_FREE'd-page-containing chunks this
        1015                 :          * arena manages. */
        1016                 :         LinkedList              chunks_madvised;
        1017                 : #endif
        1018                 :
        1019                 :         /*
        1020                 :          * In order to avoid rapid chunk allocation/deallocation when an arena
        1021                 :          * oscillates right on the cusp of needing a new chunk, cache the most
        1022                 :          * recently freed chunk.  The spare is left in the arena's chunk trees
        1023                 :          * until it is deleted.
        1024                 :          *
        1025                 :          * There is one spare chunk per arena, rather than one spare total, in
        1026                 :          * order to avoid interactions between multiple threads that could make
        1027                 :          * a single spare inadequate.
        1028                 :          */
        1029                 :         arena_chunk_t           *spare;
        1030                 :
        1031                 :         /*
        1032                 :          * Current count of pages within unused runs that are potentially
        1033                 :          * dirty, and for which madvise(... MADV_FREE) has not been called.  By
        1034                 :          * tracking this, we can institute a limit on how much dirty unused
        1035                 :          * memory is mapped for each arena.
        1036                 :          */
        1037                 :         size_t                  ndirty;
        1038                 :
        1039                 :         /*
        1040                 :          * Size/address-ordered tree of this arena's available runs.  This tree
        1041                 :          * is used for first-best-fit run allocation.
        1042                 :          */
        1043                 :         arena_avail_tree_t      runs_avail;
        1044                 :
        1045                 : #ifdef MALLOC_BALANCE
        1046                 :         /*
        1047                 :          * The arena load balancing machinery needs to keep track of how much
        1048                 :          * lock contention there is.  This value is exponentially averaged.
        1049                 :          */
        1050                 :         uint32_t                contention;
        1051                 : #endif
        1052                 :
        1053                 :         /*
        1054                 :          * bins is used to store rings of free regions of the following sizes,
        1055                 :          * assuming a 16-byte quantum, 4kB pagesize, and default MALLOC_OPTIONS.
        1056                 :          *
        1057                 :          *   bins[i] | size |
        1058                 :          *   --------+------+
        1059                 :          *        0  |    2 |
        1060                 :          *        1  |    4 |
        1061                 :          *        2  |    8 |
        1062                 :          *   --------+------+
        1063                 :          *        3  |   16 |
        1064                 :          *        4  |   32 |
        1065                 :          *        5  |   48 |
        1066                 :          *        6  |   64 |
        1067                 :          *           :      :
        1068                 :          *           :      :
        1069                 :          *       33  |  496 |
        1070                 :          *       34  |  512 |
        1071                 :          *   --------+------+
        1072                 :          *       35  | 1024 |
        1073                 :          *       36  | 2048 |
        1074                 :          *   --------+------+
        1075                 :          */
        1076                 :         arena_bin_t             bins[1]; /* Dynamically sized. */
        1077                 : };
        1078                 :
        1079                 : /******************************************************************************/
        1080                 : /*
        1081                 :  * Data.
        1082                 :  */
        1083                 :
        1084                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        1085                 : /* Number of CPUs. */
        1086                 : static unsigned         ncpus;
        1087                 : #endif
        1088                 :
        1089                 : /*
        1090                 :  * When MALLOC_STATIC_SIZES is defined most of the parameters
        1091                 :  * controlling the malloc behavior are defined as compile-time constants
        1092                 :  * for best performance and cannot be altered at runtime.
        1093                 :  */
        1094                 : #define MALLOC_STATIC_SIZES 1
        1095                 :
        1096                 : #ifdef MALLOC_STATIC_SIZES
        1097                 :
        1098                 : /*
        1099                 :  * VM page size. It must divide the runtime CPU page size or the code
        1100                 :  * will abort.
        1101                 :  */
        1102                 : #define pagesize_2pow                   ((size_t) 12)
        1103                 : #define pagesize                        ((size_t) 1 << pagesize_2pow)
        1104                 : #define pagesize_mask                   (pagesize - 1)
        1105                 :
        1106                 : /* Various quantum-related settings. */
        1107                 :
        1108                 : #define QUANTUM_DEFAULT                 ((size_t) 1 << QUANTUM_2POW_MIN)
        1109                 : static const size_t     quantum =       QUANTUM_DEFAULT;
        1110                 : static const size_t     quantum_mask =  QUANTUM_DEFAULT - 1;
        1111                 :
        1112                 : /* Various bin-related settings. */
        1113                 :
        1114                 : static const size_t     small_min =     (QUANTUM_DEFAULT >> 1) + 1;
        1115                 : static const size_t     small_max =     (size_t) SMALL_MAX_DEFAULT;
        1116                 :
        1117                 : /* Max size class for bins. */
        1118                 : static const size_t     bin_maxclass =  pagesize >> 1;
        1119                 :
        1120                 :  /* Number of (2^n)-spaced tiny bins. */
        1121                 : static const unsigned   ntbins =        (unsigned)
        1122                 :                                         (QUANTUM_2POW_MIN - TINY_MIN_2POW);
        1123                 :
        1124                 :  /* Number of quantum-spaced bins. */
        1125                 : static const unsigned   nqbins =        (unsigned)
        1126                 :                                         (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN);
        1127                 :
        1128                 : /* Number of (2^n)-spaced sub-page bins. */
        1129                 : static const unsigned   nsbins =        (unsigned)
        1130                 :                                         (pagesize_2pow -
        1131                 :                                          SMALL_MAX_2POW_DEFAULT - 1);
        1132                 :
        1133                 : #else /* !MALLOC_STATIC_SIZES */
        1134                 :
        1135                 : /* VM page size. */
        1136                 : static size_t           pagesize;
        1137                 : static size_t           pagesize_mask;
        1138                 : static size_t           pagesize_2pow;
        1139                 :
        1140                 : /* Various bin-related settings. */
        1141                 : static size_t           bin_maxclass; /* Max size class for bins. */
        1142                 : static unsigned         ntbins; /* Number of (2^n)-spaced tiny bins. */
        1143                 : static unsigned         nqbins; /* Number of quantum-spaced bins. */
        1144                 : static unsigned         nsbins; /* Number of (2^n)-spaced sub-page bins. */
        1145                 : static size_t           small_min;
        1146                 : static size_t           small_max;
        1147                 :
        1148                 : /* Various quantum-related settings. */
        1149                 : static size_t           quantum;
        1150                 : static size_t           quantum_mask; /* (quantum - 1). */
        1151                 :
        1152                 : #endif
        1153                 :
        1154                 : /* Various chunk-related settings. */
        1155                 :
        1156                 : /*
        1157                 :  * Compute the header size such that it is large enough to contain the page map
        1158                 :  * and enough nodes for the worst case: one node per non-header page plus one
        1159                 :  * extra for situations where we briefly have one more node allocated than we
        1160                 :  * will need.
        1161                 :  */
        1162                 : #define calculate_arena_header_size()                                   \
        1163                 :         (sizeof(arena_chunk_t) + sizeof(arena_chunk_map_t) * (chunk_npages - 1))
        1164                 :
        1165                 : #define calculate_arena_header_pages()                                  \
        1166                 :         ((calculate_arena_header_size() >> pagesize_2pow) +               \
        1167                 :          ((calculate_arena_header_size() & pagesize_mask) ? 1 : 0))
        1168                 :
        1169                 : /* Max size class for arenas. */
        1170                 : #define calculate_arena_maxclass()                                      \
        1171                 :         (chunksize - (arena_chunk_header_npages << pagesize_2pow))
        1172                 :
        1173                 : #ifdef MALLOC_STATIC_SIZES
        1174                 : #define CHUNKSIZE_DEFAULT               ((size_t) 1 << CHUNK_2POW_DEFAULT)
        1175                 : static const size_t     chunksize =     CHUNKSIZE_DEFAULT;
        1176                 : static const size_t     chunksize_mask =CHUNKSIZE_DEFAULT - 1;
        1177                 : static const size_t     chunk_npages =  CHUNKSIZE_DEFAULT >> pagesize_2pow;
        1178                 : #define arena_chunk_header_npages       calculate_arena_header_pages()
        1179                 : #define arena_maxclass                  calculate_arena_maxclass()
        1180                 : #else
        1181                 : static size_t           chunksize;
        1182                 : static size_t           chunksize_mask; /* (chunksize - 1). */
        1183                 : static size_t           chunk_npages;
        1184                 : static size_t           arena_chunk_header_npages;
        1185                 : static size_t           arena_maxclass; /* Max size class for arenas. */
        1186                 : #endif
        1187                 :
        1188                 : /********/
        1189                 : /*
        1190                 :  * Chunks.
        1191                 :  */
        1192                 :
        1193                 : #ifdef MALLOC_VALIDATE
        1194                 : static malloc_rtree_t *chunk_rtree;
        1195                 : #endif
        1196                 :
        1197                 : /* Protects chunk-related data structures. */
        1198                 : static malloc_mutex_t   huge_mtx;
        1199                 :
        1200                 : /* Tree of chunks that are stand-alone huge allocations. */
        1201                 : static extent_tree_t    huge;
        1202                 :
        1203                 : #ifdef MALLOC_STATS
        1204                 : /* Huge allocation statistics. */
        1205                 : static uint64_t         huge_nmalloc;
        1206                 : static uint64_t         huge_ndalloc;
        1207                 : static size_t           huge_allocated;
        1208                 : #endif
        1209                 :
        1210                 : #ifdef MALLOC_PAGEFILE
        1211                 : static char             pagefile_templ[PATH_MAX];
        1212                 : #endif
        1213                 :
        1214                 : /****************************/
        1215                 : /*
        1216                 :  * base (internal allocation).
        1217                 :  */
        1218                 :
        1219                 : /*
        1220                 :  * Current pages that are being used for internal memory allocations.  These
        1221                 :  * pages are carved up in cacheline-size quanta, so that there is no chance of
        1222                 :  * false cache line sharing.
        1223                 :  */
        1224                 : static void             *base_pages;
        1225                 : static void             *base_next_addr;
        1226                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
        1227                 : static void             *base_next_decommitted;
        1228                 : #endif
        1229                 : static void             *base_past_addr; /* Addr immediately past base_pages. */
        1230                 : static extent_node_t    *base_nodes;
        1231                 : static malloc_mutex_t   base_mtx;
        1232                 : #ifdef MALLOC_STATS
        1233                 : static size_t           base_mapped;
        1234                 : static size_t           base_committed;
        1235                 : #endif
        1236                 :
        1237                 : /********/
        1238                 : /*
        1239                 :  * Arenas.
        1240                 :  */
        1241                 :
        1242                 : /*
        1243                 :  * Arenas that are used to service external requests.  Not all elements of the
        1244                 :  * arenas array are necessarily used; arenas are created lazily as needed.
        1245                 :  */
        1246                 : static arena_t          **arenas;
        1247                 : static unsigned         narenas;
        1248                 : #ifndef NO_TLS
        1249                 : #  ifdef MALLOC_BALANCE
        1250                 : static unsigned         narenas_2pow;
        1251                 : #  else
        1252                 : static unsigned         next_arena;
        1253                 : #  endif
        1254                 : #endif
        1255                 : #ifdef MOZ_MEMORY
        1256                 : static malloc_spinlock_t arenas_lock; /* Protects arenas initialization. */
        1257                 : #else
        1258                 : static pthread_mutex_t arenas_lock; /* Protects arenas initialization. */
        1259                 : #endif
        1260                 :
        1261                 : #ifndef NO_TLS
        1262                 : /*
        1263                 :  * Map of pthread_self() --> arenas[???], used for selecting an arena to use
        1264                 :  * for allocations.
        1265                 :  */
        1266                 : #ifndef MOZ_MEMORY_WINDOWS
        1267                 : static __thread arena_t *arenas_map;
        1268                 : #endif
        1269                 : #endif
        1270                 :
        1271                 : #ifdef MALLOC_STATS
        1272                 : /* Chunk statistics. */
        1273                 : static chunk_stats_t    stats_chunks;
        1274                 : #endif
        1275                 :
        1276                 : /*******************************/
        1277                 : /*
        1278                 :  * Runtime configuration options.
        1279                 :  */
        1280                 : const char      *_malloc_options;
        1281                 :
        1282                 : #ifndef MALLOC_PRODUCTION
        1283                 : static bool     opt_abort = true;
        1284                 : #ifdef MALLOC_FILL
        1285                 : static bool     opt_junk = true;
        1286                 : #endif
        1287                 : #else
        1288                 : static bool     opt_abort = false;
        1289                 : #ifdef MALLOC_FILL
        1290                 : static bool     opt_junk = false;
        1291                 : #endif
        1292                 : #endif
        1293                 : static size_t   opt_dirty_max = DIRTY_MAX_DEFAULT;
        1294                 : #ifdef MALLOC_BALANCE
        1295                 : static uint64_t opt_balance_threshold = BALANCE_THRESHOLD_DEFAULT;
        1296                 : #endif
        1297                 : static bool     opt_print_stats = false;
        1298                 : #ifdef MALLOC_STATIC_SIZES
        1299                 : #define opt_quantum_2pow        QUANTUM_2POW_MIN
        1300                 : #define opt_small_max_2pow      SMALL_MAX_2POW_DEFAULT
        1301                 : #define opt_chunk_2pow          CHUNK_2POW_DEFAULT
        1302                 : #else
        1303                 : static size_t   opt_quantum_2pow = QUANTUM_2POW_MIN;
        1304                 : static size_t   opt_small_max_2pow = SMALL_MAX_2POW_DEFAULT;
        1305                 : static size_t   opt_chunk_2pow = CHUNK_2POW_DEFAULT;
        1306                 : #endif
        1307                 : #ifdef MALLOC_PAGEFILE
        1308                 : static bool     opt_pagefile = false;
        1309                 : #endif
        1310                 : #ifdef MALLOC_UTRACE
        1311                 : static bool     opt_utrace = false;
        1312                 : #endif
        1313                 : #ifdef MALLOC_SYSV
        1314                 : static bool     opt_sysv = false;
        1315                 : #endif
        1316                 : #ifdef MALLOC_XMALLOC
        1317                 : static bool     opt_xmalloc = false;
        1318                 : #endif
        1319                 : #ifdef MALLOC_FILL
        1320                 : static bool     opt_zero = false;
        1321                 : #endif
        1322                 : static int      opt_narenas_lshift = 0;
        1323                 :
        1324                 : #ifdef MALLOC_UTRACE
        1325                 : typedef struct {
        1326                 :         void    *p;
        1327                 :         size_t  s;
        1328                 :         void    *r;
        1329                 : } malloc_utrace_t;
        1330                 :
        1331                 : #define UTRACE(a, b, c)                                                 \
        1332                 :         if (opt_utrace) {                                               \
        1333                 :                 malloc_utrace_t ut;                                     \
        1334                 :                 ut.p = (a);                                             \
        1335                 :                 ut.s = (b);                                             \
        1336                 :                 ut.r = (c);                                             \
        1337                 :                 utrace(&ut, sizeof(ut));                            \
        1338                 :         }
        1339                 : #else
        1340                 : #define UTRACE(a, b, c)
        1341                 : #endif
        1342                 :
        1343                 : /******************************************************************************/
        1344                 : /*
        1345                 :  * Begin function prototypes for non-inline static functions.
        1346                 :  */
        1347                 :
        1348                 : static char     *umax2s(uintmax_t x, unsigned base, char *s);
        1349                 : static bool     malloc_mutex_init(malloc_mutex_t *mutex);
        1350                 : static bool     malloc_spin_init(malloc_spinlock_t *lock);
        1351                 : static void     wrtmessage(const char *p1, const char *p2, const char *p3,
        1352                 :                 const char *p4);
        1353                 : #ifdef MALLOC_STATS
        1354                 : #ifdef MOZ_MEMORY_DARWIN
        1355                 : /* Avoid namespace collision with OS X's malloc APIs. */
        1356                 : #define malloc_printf moz_malloc_printf
        1357                 : #endif
        1358                 : static void     malloc_printf(const char *format, ...);
        1359                 : #endif
        1360                 : static bool     base_pages_alloc_mmap(size_t minsize);
        1361                 : static bool     base_pages_alloc(size_t minsize);
        1362                 : static void     *base_alloc(size_t size);
        1363                 : static void     *base_calloc(size_t number, size_t size);
        1364                 : static extent_node_t *base_node_alloc(void);
        1365                 : static void     base_node_dealloc(extent_node_t *node);
        1366                 : #ifdef MALLOC_STATS
        1367                 : static void     stats_print(arena_t *arena);
        1368                 : #endif
        1369                 : static void     *pages_map(void *addr, size_t size, int pfd);
        1370                 : static void     pages_unmap(void *addr, size_t size);
        1371                 : static void     *chunk_alloc_mmap(size_t size, bool pagefile);
        1372                 : #ifdef MALLOC_PAGEFILE
        1373                 : static int      pagefile_init(size_t size);
        1374                 : static void     pagefile_close(int pfd);
        1375                 : #endif
        1376                 : static void     *chunk_alloc(size_t size, bool zero, bool pagefile);
        1377                 : static void     chunk_dealloc_mmap(void *chunk, size_t size);
        1378                 : static void     chunk_dealloc(void *chunk, size_t size);
        1379                 : #ifndef NO_TLS
        1380                 : static arena_t  *choose_arena_hard(void);
        1381                 : #endif
        1382                 : static void     arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
        1383                 :     bool large, bool zero);
        1384                 : static void arena_chunk_init(arena_t *arena, arena_chunk_t *chunk);
        1385                 : static void     arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
        1386                 : static arena_run_t *arena_run_alloc(arena_t *arena, arena_bin_t *bin,
        1387                 :     size_t size, bool large, bool zero);
        1388                 : static void     arena_purge(arena_t *arena);
        1389                 : static void     arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
        1390                 : static void     arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
        1391                 :     arena_run_t *run, size_t oldsize, size_t newsize);
        1392                 : static void     arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk,
        1393                 :     arena_run_t *run, size_t oldsize, size_t newsize, bool dirty);
        1394                 : static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
        1395                 : static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
        1396                 : static size_t arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size);
        1397                 : #ifdef MALLOC_BALANCE
        1398                 : static void     arena_lock_balance_hard(arena_t *arena);
        1399                 : #endif
        1400                 : static void     *arena_malloc_large(arena_t *arena, size_t size, bool zero);
        1401                 : static void     *arena_palloc(arena_t *arena, size_t alignment, size_t size,
        1402                 :     size_t alloc_size);
        1403                 : static size_t   arena_salloc(const void *ptr);
        1404                 : static void     arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk,
        1405                 :     void *ptr);
        1406                 : static void     arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk,
        1407                 :     void *ptr, size_t size, size_t oldsize);
        1408                 : static bool     arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk,
        1409                 :     void *ptr, size_t size, size_t oldsize);
        1410                 : static bool     arena_ralloc_large(void *ptr, size_t size, size_t oldsize);
        1411                 : static void     *arena_ralloc(void *ptr, size_t size, size_t oldsize);
        1412                 : static bool     arena_new(arena_t *arena);
        1413                 : static arena_t  *arenas_extend(unsigned ind);
        1414                 : static void     *huge_malloc(size_t size, bool zero);
        1415                 : static void     *huge_palloc(size_t alignment, size_t size);
        1416                 : static void     *huge_ralloc(void *ptr, size_t size, size_t oldsize);
        1417                 : static void     huge_dalloc(void *ptr);
        1418                 : static void     malloc_print_stats(void);
        1419                 : #ifndef MOZ_MEMORY_WINDOWS
        1420                 : static
        1421                 : #endif
        1422                 : bool            malloc_init_hard(void);
        1423                 :
        1424                 : static void     _malloc_prefork(void);
        1425                 : static void     _malloc_postfork(void);
        1426                 :
        1427                 : #ifdef MOZ_MEMORY_DARWIN
        1428                 : /*
        1429                 :  * MALLOC_ZONE_T_NOTE
        1430                 :  *
        1431                 :  * On Darwin, we hook into the memory allocator using a malloc_zone_t struct.
        1432                 :  * We must be very careful around this struct because of different behaviour on
        1433                 :  * different versions of OSX.
        1434                 :  *
        1435                 :  * Each of OSX 10.5, 10.6 and 10.7 use different versions of the struct
        1436                 :  * (with version numbers 3, 6 and 8 respectively). The binary we use on each of
        1437                 :  * these platforms will not necessarily be built using the correct SDK [1].
        1438                 :  * This means we need to statically know the correct struct size to use on all
        1439                 :  * OSX releases, and have a fallback for unknown future versions. The struct
        1440                 :  * sizes defined in osx_zone_types.h.
        1441                 :  *
        1442                 :  * For OSX 10.8 and later, we may expect the malloc_zone_t struct to change
        1443                 :  * again, and need to dynamically account for this. By simply leaving
        1444                 :  * malloc_zone_t alone, we don't quite deal with the problem, because there
        1445                 :  * remain calls to jemalloc through the mozalloc interface. We check this
        1446                 :  * dynamically on each allocation, using the CHECK_DARWIN macro and
        1447                 :  * osx_use_jemalloc.
        1448                 :  *
        1449                 :  *
        1450                 :  * [1] Mozilla is built as a universal binary on Mac, supporting i386 and
        1451                 :  *     x86_64. The i386 target is built using the 10.5 SDK, even if it runs on
        1452                 :  *     10.6. The x86_64 target is built using the 10.6 SDK, even if it runs on
        1453                 :  *     10.7 or later, or 10.5.
        1454                 :  *
        1455                 :  * FIXME:
        1456                 :  *   When later versions of OSX come out (10.8 and up), we need to check their
        1457                 :  *   malloc_zone_t versions. If they're greater than 8, we need a new version
        1458                 :  *   of malloc_zone_t adapted into osx_zone_types.h.
        1459                 :  */
        1460                 :
        1461                 : #include "osx_zone_types.h"
        1462                 :
        1463                 : #define LEOPARD_MALLOC_ZONE_T_VERSION 3
        1464                 : #define SNOW_LEOPARD_MALLOC_ZONE_T_VERSION 6
        1465                 : #define LION_MALLOC_ZONE_T_VERSION 8
        1466                 :
        1467                 : static bool osx_use_jemalloc = false;
        1468                 :
        1469                 :
        1470                 : static lion_malloc_zone l_szone;
        1471                 : static malloc_zone_t * szone = (malloc_zone_t*)(&l_szone);
        1472                 :
        1473                 : static lion_malloc_introspection l_ozone_introspect;
        1474                 : static malloc_introspection_t * const ozone_introspect =
        1475                 :         (malloc_introspection_t*)(&l_ozone_introspect);
        1476                 : static void szone2ozone(malloc_zone_t *zone, size_t size);
        1477                 : static size_t zone_version_size(int version);
        1478                 : #endif
        1479                 :
        1480                 : /*
        1481                 :  * End function prototypes.
        1482                 :  */
        1483                 : /******************************************************************************/
        1484                 :
        1485                 : /*
        1486                 :  * umax2s() provides minimal integer printing functionality, which is
        1487                 :  * especially useful for situations where allocation in vsnprintf() calls would
        1488                 :  * potentially cause deadlock.
        1489                 :  */
        1490                 : #define UMAX2S_BUFSIZE  65
        1491                 : char *
        1492               0 : umax2s(uintmax_t x, unsigned base, char *s)
        1493                 : {
        1494                 :         unsigned i;
        1495                 :
        1496               0 :         i = UMAX2S_BUFSIZE - 1;
        1497               0 :         s[i] = '\0';
        1498               0 :         switch (base) {
        1499                 :         case 10:
        1500                 :                 do {
        1501               0 :                         i--;
        1502               0 :                         s[i] = "0123456789"[x % 10];
        1503               0 :                         x /= 10;
        1504               0 :                 } while (x > 0);
        1505               0 :                 break;
        1506                 :         case 16:
        1507                 :                 do {
        1508               0 :                         i--;
        1509               0 :                         s[i] = "0123456789abcdef"[x & 0xf];
        1510               0 :                         x >>= 4;
        1511               0 :                 } while (x > 0);
        1512               0 :                 break;
        1513                 :         default:
        1514                 :                 do {
        1515               0 :                         i--;
        1516               0 :                         s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % base];
        1517               0 :                         x /= base;
        1518               0 :                 } while (x > 0);
        1519                 :         }
        1520                 :
        1521               0 :         return (&s[i]);
        1522                 : }
        1523                 :
        1524                 : static void
        1525               0 : wrtmessage(const char *p1, const char *p2, const char *p3, const char *p4)
        1526                 : {
        1527                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_WINDOWS)
        1528                 : #define _write  write
        1529                 : #endif
        1530               0 :         _write(STDERR_FILENO, p1, (unsigned int) strlen(p1));
        1531               0 :         _write(STDERR_FILENO, p2, (unsigned int) strlen(p2));
        1532               0 :         _write(STDERR_FILENO, p3, (unsigned int) strlen(p3));
        1533               0 :         _write(STDERR_FILENO, p4, (unsigned int) strlen(p4));
        1534               0 : }
        1535                 :
        1536                 : #define _malloc_message malloc_message
        1537                 :
        1538                 : void    (*_malloc_message)(const char *p1, const char *p2, const char *p3,
        1539                 :             const char *p4) = wrtmessage;
        1540                 :
        1541                 : #ifdef MALLOC_DEBUG
        1542                 : #  define assert(e) do {                                                \
        1543                 :         if (!(e)) {                                                     \
        1544                 :                 char line_buf[UMAX2S_BUFSIZE];                          \
        1545                 :                 _malloc_message(__FILE__, ":", umax2s(__LINE__, 10,   \
        1546                 :                     line_buf), ": Failed assertion: ");                       \
        1547                 :                 _malloc_message("\"", #e, "\"\n", "");                  \
        1548                 :                 abort();                                                \
        1549                 :         }                                                               \
        1550                 : } while (0)
        1551                 : #else
        1552                 : #define assert(e)
        1553                 : #endif
        1554                 :
        1555                 : /******************************************************************************/
        1556                 : /*
        1557                 :  * Begin mutex.  We can't use normal pthread mutexes in all places, because
        1558                 :  * they require malloc()ed memory, which causes bootstrapping issues in some
        1559                 :  * cases.
        1560                 :  */
        1561                 :
        1562                 : static bool
        1563           93345 : malloc_mutex_init(malloc_mutex_t *mutex)
        1564                 : {
        1565                 : #if defined(MOZ_MEMORY_WINDOWS)
        1566                 :         if (isthreaded)
        1567                 :                 if (! __crtInitCritSecAndSpinCount(mutex, _CRT_SPINCOUNT))
        1568                 :                         return (true);
        1569                 : #elif defined(MOZ_MEMORY_DARWIN)
        1570                 :         mutex->lock = OS_SPINLOCK_INIT;
        1571                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
        1572                 :         pthread_mutexattr_t attr;
        1573           93345 :         if (pthread_mutexattr_init(&attr) != 0)
        1574               0 :                 return (true);
        1575           93345 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
        1576           93345 :         if (pthread_mutex_init(mutex, &attr) != 0) {
        1577               0 :                 pthread_mutexattr_destroy(&attr);
        1578               0 :                 return (true);
        1579                 :         }
        1580           93345 :         pthread_mutexattr_destroy(&attr);
        1581                 : #elif defined(MOZ_MEMORY)
        1582                 :         if (pthread_mutex_init(mutex, NULL) != 0)
        1583                 :                 return (true);
        1584                 : #else
        1585                 :         static const spinlock_t lock = _SPINLOCK_INITIALIZER;
        1586                 :
        1587                 :         mutex->lock = lock;
        1588                 : #endif
        1589           93345 :         return (false);
        1590                 : }
        1591                 :
        1592                 : static inline void
        1593        90845182 : malloc_mutex_lock(malloc_mutex_t *mutex)
        1594                 : {
        1595                 :
        1596                 : #if defined(MOZ_MEMORY_WINDOWS)
        1597                 :         EnterCriticalSection(mutex);
        1598                 : #elif defined(MOZ_MEMORY_DARWIN)
        1599                 :         OSSpinLockLock(&mutex->lock);
        1600                 : #elif defined(MOZ_MEMORY)
        1601        90845182 :         pthread_mutex_lock(mutex);
        1602                 : #else
        1603                 :         if (isthreaded)
        1604                 :                 _SPINLOCK(&mutex->lock);
        1605                 : #endif
        1606        90845224 : }
        1607                 :
        1608                 : static inline void
        1609        90845224 : malloc_mutex_unlock(malloc_mutex_t *mutex)
        1610                 : {
        1611                 :
        1612                 : #if defined(MOZ_MEMORY_WINDOWS)
        1613                 :         LeaveCriticalSection(mutex);
        1614                 : #elif defined(MOZ_MEMORY_DARWIN)
        1615                 :         OSSpinLockUnlock(&mutex->lock);
        1616                 : #elif defined(MOZ_MEMORY)
        1617        90845224 :         pthread_mutex_unlock(mutex);
        1618                 : #else
        1619                 :         if (isthreaded)
        1620                 :                 _SPINUNLOCK(&mutex->lock);
        1621                 : #endif
        1622        90845222 : }
        1623                 :
        1624                 : static bool
        1625               0 : malloc_spin_init(malloc_spinlock_t *lock)
        1626                 : {
        1627                 : #if defined(MOZ_MEMORY_WINDOWS)
        1628                 :         if (isthreaded)
        1629                 :                 if (! __crtInitCritSecAndSpinCount(lock, _CRT_SPINCOUNT))
        1630                 :                         return (true);
        1631                 : #elif defined(MOZ_MEMORY_DARWIN)
        1632                 :         lock->lock = OS_SPINLOCK_INIT;
        1633                 : #elif defined(MOZ_MEMORY_LINUX) && !defined(MOZ_MEMORY_ANDROID)
        1634                 :         pthread_mutexattr_t attr;
        1635               0 :         if (pthread_mutexattr_init(&attr) != 0)
        1636               0 :                 return (true);
        1637               0 :         pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP);
        1638               0 :         if (pthread_mutex_init(lock, &attr) != 0) {
        1639               0 :                 pthread_mutexattr_destroy(&attr);
        1640               0 :                 return (true);
        1641                 :         }
        1642               0 :         pthread_mutexattr_destroy(&attr);
        1643                 : #elif defined(MOZ_MEMORY)
        1644                 :         if (pthread_mutex_init(lock, NULL) != 0)
        1645                 :                 return (true);
        1646                 : #else
        1647                 :         lock->lock = _SPINLOCK_INITIALIZER;
        1648                 : #endif
        1649               0 :         return (false);
        1650                 : }
        1651                 :
        1652                 : static inline void
        1653               0 : malloc_spin_lock(malloc_spinlock_t *lock)
        1654                 : {
        1655                 :
        1656                 : #if defined(MOZ_MEMORY_WINDOWS)
        1657                 :         EnterCriticalSection(lock);
        1658                 : #elif defined(MOZ_MEMORY_DARWIN)
        1659                 :         OSSpinLockLock(&lock->lock);
        1660                 : #elif defined(MOZ_MEMORY)
        1661               0 :         pthread_mutex_lock(lock);
        1662                 : #else
        1663                 :         if (isthreaded)
        1664                 :                 _SPINLOCK(&lock->lock);
        1665                 : #endif
        1666               0 : }
        1667                 :
        1668                 : static inline void
        1669               0 : malloc_spin_unlock(malloc_spinlock_t *lock)
        1670                 : {
        1671                 : #if defined(MOZ_MEMORY_WINDOWS)
        1672                 :         LeaveCriticalSection(lock);
        1673                 : #elif defined(MOZ_MEMORY_DARWIN)
        1674                 :         OSSpinLockUnlock(&lock->lock);
        1675                 : #elif defined(MOZ_MEMORY)
        1676               0 :         pthread_mutex_unlock(lock);
        1677                 : #else
        1678                 :         if (isthreaded)
        1679                 :                 _SPINUNLOCK(&lock->lock);
        1680                 : #endif
        1681               0 : }
        1682                 :
        1683                 : /*
        1684                 :  * End mutex.
        1685                 :  */
        1686                 : /******************************************************************************/
        1687                 : /*
        1688                 :  * Begin spin lock.  Spin locks here are actually adaptive mutexes that block
        1689                 :  * after a period of spinning, because unbounded spinning would allow for
        1690                 :  * priority inversion.
        1691                 :  */
        1692                 :
        1693                 : #if defined(MOZ_MEMORY) && !defined(MOZ_MEMORY_DARWIN)
        1694                 : #  define       malloc_spin_init        malloc_mutex_init
        1695                 : #  define       malloc_spin_lock        malloc_mutex_lock
        1696                 : #  define       malloc_spin_unlock      malloc_mutex_unlock
        1697                 : #endif
        1698                 :
        1699                 : #ifndef MOZ_MEMORY
        1700                 : /*
        1701                 :  * We use an unpublished interface to initialize pthread mutexes with an
        1702                 :  * allocation callback, in order to avoid infinite recursion.
        1703                 :  */
        1704                 : int     _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
        1705                 :     void *(calloc_cb)(size_t, size_t));
        1706                 :
        1707                 : __weak_reference(_pthread_mutex_init_calloc_cb_stub,
        1708                 :     _pthread_mutex_init_calloc_cb);
        1709                 :
        1710                 : int
        1711                 : _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
        1712                 :     void *(calloc_cb)(size_t, size_t))
        1713                 : {
        1714                 :
        1715                 :         return (0);
        1716                 : }
        1717                 :
        1718                 : static bool
        1719                 : malloc_spin_init(pthread_mutex_t *lock)
        1720                 : {
        1721                 :
        1722                 :         if (_pthread_mutex_init_calloc_cb(lock, base_calloc) != 0)
        1723                 :                 return (true);
        1724                 :
        1725                 :         return (false);
        1726                 : }
        1727                 :
        1728                 : static inline unsigned
        1729                 : malloc_spin_lock(pthread_mutex_t *lock)
        1730                 : {
        1731                 :         unsigned ret = 0;
        1732                 :
        1733                 :         if (isthreaded) {
        1734                 :                 if (_pthread_mutex_trylock(lock) != 0) {
        1735                 :                         unsigned i;
        1736                 :                         volatile unsigned j;
        1737                 :
        1738                 :                         /* Exponentially back off. */
        1739                 :                         for (i = 1; i <= SPIN_LIMIT_2POW; i++) {
        1740                 :                                 for (j = 0; j < (1U << i); j++)
        1741                 :                                         ret++;
        1742                 :
        1743                 :                                 CPU_SPINWAIT;
        1744                 :                                 if (_pthread_mutex_trylock(lock) == 0)
        1745                 :                                         return (ret);
        1746                 :                         }
        1747                 :
        1748                 :                         /*
        1749                 :                          * Spinning failed.  Block until the lock becomes
        1750                 :                          * available, in order to avoid indefinite priority
        1751                 :                          * inversion.
        1752                 :                          */
        1753                 :                         _pthread_mutex_lock(lock);
        1754                 :                         assert((ret << BLOCK_COST_2POW) != 0);
        1755                 :                         return (ret << BLOCK_COST_2POW);
        1756                 :                 }
        1757                 :         }
        1758                 :
        1759                 :         return (ret);
        1760                 : }
        1761                 :
        1762                 : static inline void
        1763                 : malloc_spin_unlock(pthread_mutex_t *lock)
        1764                 : {
        1765                 :
        1766                 :         if (isthreaded)
        1767                 :                 _pthread_mutex_unlock(lock);
        1768                 : }
        1769                 : #endif
        1770                 :
        1771                 : /*
        1772                 :  * End spin lock.
        1773                 :  */
        1774                 : /******************************************************************************/
        1775                 : /*
        1776                 :  * Begin Utility functions/macros.
        1777                 :  */
        1778                 :
        1779                 : /* Return the chunk address for allocation address a. */
        1780                 : #define CHUNK_ADDR2BASE(a)                                              \
        1781                 :         ((void *)((uintptr_t)(a) & ~chunksize_mask))
        1782                 :
        1783                 : /* Return the chunk offset of address a. */
        1784                 : #define CHUNK_ADDR2OFFSET(a)                                            \
        1785                 :         ((size_t)((uintptr_t)(a) & chunksize_mask))
        1786                 :
        1787                 : /* Return the smallest chunk multiple that is >= s. */
        1788                 : #define CHUNK_CEILING(s)                                                \
        1789                 :         (((s) + chunksize_mask) & ~chunksize_mask)
        1790                 :
        1791                 : /* Return the smallest cacheline multiple that is >= s. */
        1792                 : #define CACHELINE_CEILING(s)                                            \
        1793                 :         (((s) + (CACHELINE - 1)) & ~(CACHELINE - 1))
        1794                 :
        1795                 : /* Return the smallest quantum multiple that is >= a. */
        1796                 : #define QUANTUM_CEILING(a)                                              \
        1797                 :         (((a) + quantum_mask) & ~quantum_mask)
        1798                 :
        1799                 : /* Return the smallest pagesize multiple that is >= s. */
        1800                 : #define PAGE_CEILING(s)                                                 \
        1801                 :         (((s) + pagesize_mask) & ~pagesize_mask)
        1802                 :
        1803                 : /* Compute the smallest power of 2 that is >= x. */
        1804                 : static inline size_t
        1805        14496575 : pow2_ceil(size_t x)
        1806                 : {
        1807                 :
        1808        14496575 :         x--;
        1809        14496575 :         x |= x >> 1;
        1810        14496575 :         x |= x >> 2;
        1811        14496575 :         x |= x >> 4;
        1812        14496575 :         x |= x >> 8;
        1813        14496575 :         x |= x >> 16;
        1814                 : #if (SIZEOF_PTR == 8)
        1815                 :         x |= x >> 32;
        1816                 : #endif
        1817        14496575 :         x++;
        1818        14496575 :         return (x);
        1819                 : }
        1820                 :
        1821                 : #ifdef MALLOC_BALANCE
        1822                 : /*
        1823                 :  * Use a simple linear congruential pseudo-random number generator:
        1824                 :  *
        1825                 :  *   prn(y) = (a*x + c) % m
        1826                 :  *
        1827                 :  * where the following constants ensure maximal period:
        1828                 :  *
        1829                 :  *   a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4.
        1830                 :  *   c == Odd number (relatively prime to 2^n).
        1831                 :  *   m == 2^32
        1832                 :  *
        1833                 :  * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints.
        1834                 :  *
        1835                 :  * This choice of m has the disadvantage that the quality of the bits is
        1836                 :  * proportional to bit position.  For example. the lowest bit has a cycle of 2,
        1837                 :  * the next has a cycle of 4, etc.  For this reason, we prefer to use the upper
        1838                 :  * bits.
        1839                 :  */
        1840                 : #  define PRN_DEFINE(suffix, var, a, c)                                 \
        1841                 : static inline void                                                      \
        1842                 : sprn_##suffix(uint32_t seed)                                            \
        1843                 : {                                                                       \
        1844                 :         var = seed;                                                     \
        1845                 : }                                                                       \
        1846                 :                                                                         \
        1847                 : static inline uint32_t                                                  \
        1848                 : prn_##suffix(uint32_t lg_range)                                         \
        1849                 : {                                                                       \
        1850                 :         uint32_t ret, x;                                                \
        1851                 :                                                                         \
        1852                 :         assert(lg_range > 0);                                                \
        1853                 :         assert(lg_range <= 32);                                              \
        1854                 :                                                                         \
        1855                 :         x = (var * (a)) + (c);                                          \
        1856                 :         var = x;                                                        \
        1857                 :         ret = x >> (32 - lg_range);                                       \
        1858                 :                                                                         \
        1859                 :         return (ret);                                                   \
        1860                 : }
        1861                 : #  define SPRN(suffix, seed)    sprn_##suffix(seed)
        1862                 : #  define PRN(suffix, lg_range) prn_##suffix(lg_range)
        1863                 : #endif
        1864                 :
        1865                 : #ifdef MALLOC_BALANCE
        1866                 : /* Define the PRNG used for arena assignment. */
        1867                 : static __thread uint32_t balance_x;
        1868                 : PRN_DEFINE(balance, balance_x, 1297, 1301)
        1869                 : #endif
        1870                 :
        1871                 : #ifdef MALLOC_UTRACE
        1872                 : static int
        1873               0 : utrace(const void *addr, size_t len)
        1874                 : {
        1875               0 :         malloc_utrace_t *ut = (malloc_utrace_t *)addr;
        1876                 :         char buf_a[UMAX2S_BUFSIZE];
        1877                 :         char buf_b[UMAX2S_BUFSIZE];
        1878                 :
        1879               0 :         assert(len == sizeof(malloc_utrace_t));
        1880                 :
        1881               0 :         if (ut->p == NULL && ut->s == 0 && ut->r == NULL) {
        1882               0 :                 _malloc_message(
        1883               0 :                     umax2s(getpid(), 10, buf_a),
        1884                 :                     " x USER malloc_init()\n", "", "");
        1885               0 :         } else if (ut->p == NULL && ut->r != NULL) {
        1886               0 :                 _malloc_message(
        1887               0 :                     umax2s(getpid(), 10, buf_a),
        1888                 :                     " x USER 0x",
        1889               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
        1890                 :                     " = malloc(");
        1891               0 :                 _malloc_message(
        1892               0 :                     umax2s(ut->s, 10, buf_a),
        1893                 :                     ")\n", "", "");
        1894               0 :         } else if (ut->p != NULL && ut->r != NULL) {
        1895               0 :                 _malloc_message(
        1896               0 :                     umax2s(getpid(), 10, buf_a),
        1897                 :                     " x USER 0x",
        1898               0 :                     umax2s((uintptr_t)ut->r, 16, buf_b),
        1899                 :                     " = realloc(0x");
        1900               0 :                 _malloc_message(
        1901               0 :                     umax2s((uintptr_t)ut->p, 16, buf_a),
        1902                 :                     ", ",
        1903               0 :                     umax2s(ut->s, 10, buf_b),
        1904                 :                     ")\n");
        1905                 :         } else {
        1906               0 :                 _malloc_message(
        1907               0 :                     umax2s(getpid(), 10, buf_a),
        1908                 :                     " x USER free(0x",
        1909               0 :                     umax2s((uintptr_t)ut->p, 16, buf_b),
        1910                 :                     ")\n");
        1911                 :         }
        1912                 :
        1913               0 :         return (0);
        1914                 : }
        1915                 : #endif
        1916                 :
        1917                 : static inline const char *
        1918               0 : _getprogname(void)
        1919                 : {
        1920                 :
        1921               0 :         return ("<jemalloc>");
        1922                 : }
        1923                 :
        1924                 : #ifdef MALLOC_STATS
        1925                 : /*
        1926                 :  * Print to stderr in such a way as to (hopefully) avoid memory allocation.
        1927                 :  */
        1928                 : static void
        1929               0 : malloc_printf(const char *format, ...)
        1930                 : {
        1931                 :         char buf[4096];
        1932                 :         va_list ap;
        1933                 :
        1934               0 :         va_start(ap, format);
        1935               0 :         vsnprintf(buf, sizeof(buf), format, ap);
        1936               0 :         va_end(ap);
        1937               0 :         _malloc_message(buf, "", "", "");
        1938               0 : }
        1939                 : #endif
        1940                 :
        1941                 : /******************************************************************************/
        1942                 :
        1943                 : static inline void
        1944               0 : pages_decommit(void *addr, size_t size)
        1945                 : {
        1946                 :
        1947                 : #ifdef MOZ_MEMORY_WINDOWS
        1948                 :         VirtualFree(addr, size, MEM_DECOMMIT);
        1949                 : #else
        1950               0 :         if (mmap(addr, size, PROT_NONE, MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1,
        1951                 :             0) == MAP_FAILED)
        1952               0 :                 abort();
        1953                 : #endif
        1954               0 : }
        1955                 :
        1956                 : static inline void
        1957               0 : pages_commit(void *addr, size_t size)
        1958                 : {
        1959                 :
        1960                 : #  ifdef MOZ_MEMORY_WINDOWS
        1961                 :         VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE);
        1962                 : #  else
        1963               0 :         if (mmap(addr, size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_PRIVATE |
        1964                 :             MAP_ANON, -1, 0) == MAP_FAILED)
        1965               0 :                 abort();
        1966                 : #  endif
        1967               0 : }
        1968                 :
        1969                 : static bool
        1970           18669 : base_pages_alloc_mmap(size_t minsize)
        1971                 : {
        1972                 :         bool ret;
        1973                 :         size_t csize;
        1974                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
        1975                 :         size_t pminsize;
        1976                 : #endif
        1977                 :         int pfd;
        1978                 :
        1979           18669 :         assert(minsize != 0);
        1980           18669 :         csize = CHUNK_CEILING(minsize);
        1981                 : #ifdef MALLOC_PAGEFILE
        1982                 :         if (opt_pagefile) {
        1983                 :                 pfd = pagefile_init(csize);
        1984                 :                 if (pfd == -1)
        1985                 :                         return (true);
        1986                 :         } else
        1987                 : #endif
        1988           18669 :                 pfd = -1;
        1989           18669 :         base_pages = pages_map(NULL, csize, pfd);
        1990           18669 :         if (base_pages == NULL) {
        1991               0 :                 ret = true;
        1992               0 :                 goto RETURN;
        1993                 :         }
        1994           18669 :         base_next_addr = base_pages;
        1995           18669 :         base_past_addr = (void *)((uintptr_t)base_pages + csize);
        1996                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
        1997                 :         /*
        1998                 :          * Leave enough pages for minsize committed, since otherwise they would
        1999                 :          * have to be immediately recommitted.
        2000                 :          */
        2001           18669 :         pminsize = PAGE_CEILING(minsize);
        2002           18669 :         base_next_decommitted = (void *)((uintptr_t)base_pages + pminsize);
        2003                 : #  if defined(MALLOC_DECOMMIT)
        2004                 :         if (pminsize < csize)
        2005                 :                 pages_decommit(base_next_decommitted, csize - pminsize);
        2006                 : #  endif
        2007                 : #  ifdef MALLOC_STATS
        2008           18669 :         base_mapped += csize;
        2009           18669 :         base_committed += pminsize;
        2010                 : #  endif
        2011                 : #endif
        2012                 :
        2013           18669 :         ret = false;
        2014                 : RETURN:
        2015                 : #ifdef MALLOC_PAGEFILE
        2016                 :         if (pfd != -1)
        2017                 :                 pagefile_close(pfd);
        2018                 : #endif
        2019           18669 :         return (false);
        2020                 : }
        2021                 :
        2022                 : static bool
        2023           18669 : base_pages_alloc(size_t minsize)
        2024                 : {
        2025                 :
        2026           18669 :         if (base_pages_alloc_mmap(minsize) == false)
        2027           18669 :                 return (false);
        2028                 :
        2029               0 :         return (true);
        2030                 : }
        2031                 :
        2032                 : static void *
        2033           75275 : base_alloc(size_t size)
        2034                 : {
        2035                 :         void *ret;
        2036                 :         size_t csize;
        2037                 :
        2038                 :         /* Round size up to nearest multiple of the cacheline size. */
        2039           75275 :         csize = CACHELINE_CEILING(size);
        2040                 :
        2041           75275 :         malloc_mutex_lock(&base_mtx);
        2042                 :         /* Make sure there's enough space for the allocation. */
        2043           75275 :         if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) {
        2044           18669 :                 if (base_pages_alloc(csize)) {
        2045               0 :                         malloc_mutex_unlock(&base_mtx);
        2046               0 :                         return (NULL);
        2047                 :                 }
        2048                 :         }
        2049                 :         /* Allocate. */
        2050           75275 :         ret = base_next_addr;
        2051           75275 :         base_next_addr = (void *)((uintptr_t)base_next_addr + csize);
        2052                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS)
        2053                 :         /* Make sure enough pages are committed for the new allocation. */
        2054           75275 :         if ((uintptr_t)base_next_addr > (uintptr_t)base_next_decommitted) {
        2055           18669 :                 void *pbase_next_addr =
        2056           18669 :                     (void *)(PAGE_CEILING((uintptr_t)base_next_addr));
        2057                 :
        2058                 : #  ifdef MALLOC_DECOMMIT
        2059                 :                 pages_commit(base_next_decommitted, (uintptr_t)pbase_next_addr -
        2060                 :                     (uintptr_t)base_next_decommitted);
        2061                 : #  endif
        2062           18669 :                 base_next_decommitted = pbase_next_addr;
        2063                 : #  ifdef MALLOC_STATS
        2064           37338 :                 base_committed += (uintptr_t)pbase_next_addr -
        2065           18669 :                     (uintptr_t)base_next_decommitted;
        2066                 : #  endif
        2067                 :         }
        2068                 : #endif
        2069           75275 :         malloc_mutex_unlock(&base_mtx);
        2070                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
        2071                 :
        2072           75275 :         return (ret);
        2073                 : }
        2074                 :
        2075                 : static void *
        2076           37338 : base_calloc(size_t number, size_t size)
        2077                 : {
        2078                 :         void *ret;
        2079                 :
        2080           37338 :         ret = base_alloc(number * size);
        2081                 : #ifdef MALLOC_VALGRIND
        2082                 :         if (ret != NULL) {
        2083                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
        2084                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, true);
        2085                 :         }
        2086                 : #endif
        2087           37338 :         memset(ret, 0, number * size);
        2088                 :
        2089           37338 :         return (ret);
        2090                 : }
        2091                 :
        2092                 : static extent_node_t *
        2093            4554 : base_node_alloc(void)
        2094                 : {
        2095                 :         extent_node_t *ret;
        2096                 :
        2097            4554 :         malloc_mutex_lock(&base_mtx);
        2098            4554 :         if (base_nodes != NULL) {
        2099            3955 :                 ret = base_nodes;
        2100            3955 :                 base_nodes = *(extent_node_t **)ret;
        2101                 :                 VALGRIND_FREELIKE_BLOCK(ret, 0);
        2102                 :                 VALGRIND_MALLOCLIKE_BLOCK(ret, sizeof(extent_node_t), 0, false);
        2103            3955 :                 malloc_mutex_unlock(&base_mtx);
        2104                 :         } else {
        2105             599 :                 malloc_mutex_unlock(&base_mtx);
        2106             599 :                 ret = (extent_node_t *)base_alloc(sizeof(extent_node_t));
        2107                 :         }
        2108                 :
        2109            4554 :         return (ret);
        2110                 : }
        2111                 :
        2112                 : static void
        2113            4554 : base_node_dealloc(extent_node_t *node)
        2114                 : {
        2115                 :
        2116            4554 :         malloc_mutex_lock(&base_mtx);
        2117                 :         VALGRIND_FREELIKE_BLOCK(node, 0);
        2118                 :         VALGRIND_MALLOCLIKE_BLOCK(node, sizeof(extent_node_t *), 0, false);
        2119            4554 :         *(extent_node_t **)node = base_nodes;
        2120            4554 :         base_nodes = node;
        2121            4554 :         malloc_mutex_unlock(&base_mtx);
        2122            4554 : }
        2123                 :
        2124                 : /******************************************************************************/
        2125                 :
        2126                 : #ifdef MALLOC_STATS
        2127                 : static void
        2128               0 : stats_print(arena_t *arena)
        2129                 : {
        2130                 :         unsigned i, gap_start;
        2131                 :
        2132                 : #ifdef MOZ_MEMORY_WINDOWS
        2133                 :         malloc_printf("dirty: %Iu page%s dirty, %I64u sweep%s,"
        2134                 :             " %I64u madvise%s, %I64u page%s purged\n",
        2135                 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
        2136                 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
        2137                 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
        2138                 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
        2139                 : #  ifdef MALLOC_DECOMMIT
        2140                 :         malloc_printf("decommit: %I64u decommit%s, %I64u commit%s,"
        2141                 :             " %I64u page%s decommitted\n",
        2142                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
        2143                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
        2144                 :             arena->stats.decommitted,
        2145                 :             (arena->stats.decommitted == 1) ? "" : "s");
        2146                 : #  endif
        2147                 :
        2148                 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
        2149                 :         malloc_printf("small:   %12Iu %12I64u %12I64u\n",
        2150                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
        2151                 :             arena->stats.ndalloc_small);
        2152                 :         malloc_printf("large:   %12Iu %12I64u %12I64u\n",
        2153                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
        2154                 :             arena->stats.ndalloc_large);
        2155                 :         malloc_printf("total:   %12Iu %12I64u %12I64u\n",
        2156                 :             arena->stats.allocated_small + arena->stats.allocated_large,
        2157                 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
        2158                 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
        2159                 :         malloc_printf("mapped:  %12Iu\n", arena->stats.mapped);
        2160                 : #else
        2161               0 :         malloc_printf("dirty: %zu page%s dirty, %llu sweep%s,"
        2162                 :             " %llu madvise%s, %llu page%s purged\n",
        2163               0 :             arena->ndirty, arena->ndirty == 1 ? "" : "s",
        2164               0 :             arena->stats.npurge, arena->stats.npurge == 1 ? "" : "s",
        2165               0 :             arena->stats.nmadvise, arena->stats.nmadvise == 1 ? "" : "s",
        2166               0 :             arena->stats.purged, arena->stats.purged == 1 ? "" : "s");
        2167                 : #  ifdef MALLOC_DECOMMIT
        2168                 :         malloc_printf("decommit: %llu decommit%s, %llu commit%s,"
        2169                 :             " %llu page%s decommitted\n",
        2170                 :             arena->stats.ndecommit, (arena->stats.ndecommit == 1) ? "" : "s",
        2171                 :             arena->stats.ncommit, (arena->stats.ncommit == 1) ? "" : "s",
        2172                 :             arena->stats.decommitted,
        2173                 :             (arena->stats.decommitted == 1) ? "" : "s");
        2174                 : #  endif
        2175                 :
        2176               0 :         malloc_printf("            allocated      nmalloc      ndalloc\n");
        2177               0 :         malloc_printf("small:   %12zu %12llu %12llu\n",
        2178                 :             arena->stats.allocated_small, arena->stats.nmalloc_small,
        2179                 :             arena->stats.ndalloc_small);
        2180               0 :         malloc_printf("large:   %12zu %12llu %12llu\n",
        2181                 :             arena->stats.allocated_large, arena->stats.nmalloc_large,
        2182                 :             arena->stats.ndalloc_large);
        2183               0 :         malloc_printf("total:   %12zu %12llu %12llu\n",
        2184               0 :             arena->stats.allocated_small + arena->stats.allocated_large,
        2185               0 :             arena->stats.nmalloc_small + arena->stats.nmalloc_large,
        2186               0 :             arena->stats.ndalloc_small + arena->stats.ndalloc_large);
        2187               0 :         malloc_printf("mapped:  %12zu\n", arena->stats.mapped);
        2188                 : #endif
        2189               0 :         malloc_printf("bins:     bin   size regs pgs  requests   newruns"
        2190                 :             "    reruns maxruns curruns\n");
        2191               0 :         for (i = 0, gap_start = UINT_MAX; i < ntbins + nqbins + nsbins; i++) {
        2192               0 :                 if (arena->bins[i].stats.nrequests == 0) {
        2193               0 :                         if (gap_start == UINT_MAX)
        2194               0 :                                 gap_start = i;
        2195                 :                 } else {
        2196               0 :                         if (gap_start != UINT_MAX) {
        2197               0 :                                 if (i > gap_start + 1) {
        2198                 :                                         /* Gap of more than one size class. */
        2199               0 :                                         malloc_printf("[%u..%u]\n",
        2200                 :                                             gap_start, i - 1);
        2201                 :                                 } else {
        2202                 :                                         /* Gap of one size class. */
        2203               0 :                                         malloc_printf("[%u]\n", gap_start);
        2204                 :                                 }
        2205               0 :                                 gap_start = UINT_MAX;
        2206                 :                         }
        2207               0 :                         malloc_printf(
        2208                 : #if defined(MOZ_MEMORY_WINDOWS)
        2209                 :                             "%13u %1s %4u %4u %3u %9I64u %9I64u"
        2210                 :                             " %9I64u %7u %7u\n",
        2211                 : #else
        2212                 :                             "%13u %1s %4u %4u %3u %9llu %9llu"
        2213                 :                             " %9llu %7lu %7lu\n",
        2214                 : #endif
        2215                 :                             i,
        2216               0 :                             i < ntbins ? "T" : i < ntbins + nqbins ? "Q" : "S",
        2217                 :                             arena->bins[i].reg_size,
        2218                 :                             arena->bins[i].nregs,
        2219               0 :                             arena->bins[i].run_size >> pagesize_2pow,
        2220                 :                             arena->bins[i].stats.nrequests,
        2221                 :                             arena->bins[i].stats.nruns,
        2222                 :                             arena->bins[i].stats.reruns,
        2223                 :                             arena->bins[i].stats.highruns,
        2224                 :                             arena->bins[i].stats.curruns);
        2225                 :                 }
        2226                 :         }
        2227               0 :         if (gap_start != UINT_MAX) {
        2228               0 :                 if (i > gap_start + 1) {
        2229                 :                         /* Gap of more than one size class. */
        2230               0 :                         malloc_printf("[%u..%u]\n", gap_start, i - 1);
        2231                 :                 } else {
        2232                 :                         /* Gap of one size class. */
        2233               0 :                         malloc_printf("[%u]\n", gap_start);
        2234                 :                 }
        2235                 :         }
        2236               0 : }
        2237                 : #endif
        2238                 :
        2239                 : /*
        2240                 :  * End Utility functions/macros.
        2241                 :  */
        2242                 : /******************************************************************************/
        2243                 : /*
        2244                 :  * Begin extent tree code.
        2245                 :  */
        2246                 :
        2247                 : static inline int
        2248               0 : extent_szad_comp(extent_node_t *a, extent_node_t *b)
        2249                 : {
        2250                 :         int ret;
        2251               0 :         size_t a_size = a->size;
        2252               0 :         size_t b_size = b->size;
        2253                 :
        2254               0 :         ret = (a_size > b_size) - (a_size < b_size);
        2255               0 :         if (ret == 0) {
        2256               0 :                 uintptr_t a_addr = (uintptr_t)a->addr;
        2257               0 :                 uintptr_t b_addr = (uintptr_t)b->addr;
        2258                 :
        2259               0 :                 ret = (a_addr > b_addr) - (a_addr < b_addr);
        2260                 :         }
        2261                 :
        2262               0 :         return (ret);
        2263                 : }
        2264                 :
        2265                 : /* Wrap red-black tree macros in functions. */
        2266               0 : rb_wrap(static, extent_tree_szad_, extent_tree_t, extent_node_t,
        2267               0 :     link_szad, extent_szad_comp)
        2268                 :
        2269                 : static inline int
        2270           28261 : extent_ad_comp(extent_node_t *a, extent_node_t *b)
        2271                 : {
        2272           28261 :         uintptr_t a_addr = (uintptr_t)a->addr;
        2273           28261 :         uintptr_t b_addr = (uintptr_t)b->addr;
        2274                 :
        2275           28261 :         return ((a_addr > b_addr) - (a_addr < b_addr));
        2276                 : }
        2277                 :
        2278                 : /* Wrap red-black tree macros in functions. */
        2279           33897 : rb_wrap(static, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad,
        2280           33897 :     extent_ad_comp)
        2281                 :
        2282                 : /*
        2283                 :  * End extent tree code.
        2284                 :  */
        2285                 : /******************************************************************************/
        2286                 : /*
        2287                 :  * Begin chunk management functions.
        2288                 :  */
        2289                 :
        2290                 : #ifdef MOZ_MEMORY_WINDOWS
        2291                 :
        2292                 : static void *
        2293                 : pages_map(void *addr, size_t size, int pfd)
        2294                 : {
        2295                 :         void *ret = NULL;
        2296                 :         ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
        2297                 :             PAGE_READWRITE);
        2298                 :         return (ret);
        2299                 : }
        2300                 :
        2301                 : static void
        2302                 : pages_unmap(void *addr, size_t size)
        2303                 : {
        2304                 :         if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
        2305                 :                 _malloc_message(_getprogname(),
        2306                 :                     ": (malloc) Error in VirtualFree()\n", "", "");
        2307                 :                 if (opt_abort)
        2308                 :                         abort();
        2309                 :         }
        2310                 : }
        2311                 : #else
        2312                 : #ifdef JEMALLOC_USES_MAP_ALIGN
        2313                 : static void *
        2314                 : pages_map_align(size_t size, int pfd, size_t alignment)
        2315                 : {
        2316                 :         void *ret;
        2317                 :
        2318                 :         /*
        2319                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
        2320                 :          * of existing mappings, and we only want to create new mappings.
        2321                 :          */
        2322                 : #ifdef MALLOC_PAGEFILE
        2323                 :         if (pfd != -1) {
        2324                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
        2325                 :                     MAP_NOSYNC | MAP_ALIGN, pfd, 0);
        2326                 :         } else
        2327                 : #endif
        2328                 :                {
        2329                 :                 ret = mmap((void *)alignment, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
        2330                 :                     MAP_NOSYNC | MAP_ALIGN | MAP_ANON, -1, 0);
        2331                 :         }
        2332                 :         assert(ret != NULL);
        2333                 :
        2334                 :         if (ret == MAP_FAILED)
        2335                 :                 ret = NULL;
        2336                 :         return (ret);
        2337                 : }
        2338                 : #endif
        2339                 :
        2340                 : static void *
        2341           64774 : pages_map(void *addr, size_t size, int pfd)
        2342                 : {
        2343                 :         void *ret;
        2344                 : #if defined(__ia64__)
        2345                 :         /*
        2346                 :          * The JS engine assumes that all allocated pointers have their high 17 bits clear,
        2347                 :          * which ia64's mmap doesn't support directly. However, we can emulate it by passing
        2348                 :          * mmap an "addr" parameter with those bits clear. The mmap will return that address,
        2349                 :          * or the nearest available memory above that address, providing a near-guarantee
        2350                 :          * that those bits are clear. If they are not, we return NULL below to indicate
        2351                 :          * out-of-memory.
        2352                 :          *
        2353                 :          * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual
        2354                 :          * address space.
        2355                 :          *
        2356                 :          * See Bug 589735 for more information.
        2357                 :          */
        2358                 :         bool check_placement = true;
        2359                 :         if (addr == NULL) {
        2360                 :                 addr = (void*)0x0000070000000000;
        2361                 :                 check_placement = false;
        2362                 :         }
        2363                 : #endif
        2364                 :
        2365                 :         /*
        2366                 :          * We don't use MAP_FIXED here, because it can cause the *replacement*
        2367                 :          * of existing mappings, and we only want to create new mappings.
        2368                 :          */
        2369                 : #ifdef MALLOC_PAGEFILE
        2370                 :         if (pfd != -1) {
        2371                 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
        2372                 :                     MAP_NOSYNC, pfd, 0);
        2373                 :         } else
        2374                 : #endif
        2375                 :                {
        2376           64774 :                 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE |
        2377                 :                     MAP_ANON, -1, 0);
        2378                 :         }
        2379           64774 :         assert(ret != NULL);
        2380                 :
        2381           64774 :         if (ret == MAP_FAILED) {
        2382              18 :                 ret = NULL;
        2383                 :         }
        2384                 : #if defined(__ia64__)
        2385                 :         /*
        2386                 :          * If the allocated memory doesn't have its upper 17 bits clear, consider it
        2387                 :          * as out of memory.
        2388                 :         */
        2389                 :         else if ((long long)ret & 0xffff800000000000) {
        2390                 :                 munmap(ret, size);
        2391                 :                 ret = NULL;
        2392                 :         }
        2393                 :         /* If the caller requested a specific memory location, verify that's what mmap returned. */
        2394                 :         else if (check_placement && ret != addr) {
        2395                 : #else
        2396           64756 :         else if (addr != NULL && ret != addr) {
        2397                 : #endif
        2398                 :                 /*
        2399                 :                  * We succeeded in mapping memory, but not in the right place.
        2400                 :                  */
        2401               0 :                 if (munmap(ret, size) == -1) {
        2402                 :                         char buf[STRERROR_BUF];
        2403                 :
        2404               0 :                         strerror_r(errno, buf, sizeof(buf));
        2405               0 :                         _malloc_message(_getprogname(),
        2406                 :                             ": (malloc) Error in munmap(): ", buf, "\n");
        2407               0 :                         if (opt_abort)
        2408               0 :                                 abort();
        2409                 :                 }
        2410               0 :                 ret = NULL;
        2411                 :         }
        2412                 :
        2413                 : #if defined(__ia64__)
        2414                 :         assert(ret == NULL || (!check_placement && ret != NULL)
        2415                 :             || (check_placement && ret == addr));
        2416                 : #else
        2417           64774 :         assert(ret == NULL || (addr == NULL && ret != addr)
        2418                 :             || (addr != NULL && ret == addr));
        2419                 : #endif
        2420           64774 :         return (ret);
        2421                 : }
        2422                 :
        2423                 : static void
        2424           27462 : pages_unmap(void *addr, size_t size)
        2425                 : {
        2426                 :
        2427           27462 :         if (munmap(addr, size) == -1) {
        2428                 :                 char buf[STRERROR_BUF];
        2429                 :
        2430               0 :                 strerror_r(errno, buf, sizeof(buf));
        2431               0 :                 _malloc_message(_getprogname(),
        2432                 :                     ": (malloc) Error in munmap(): ", buf, "\n");
        2433               0 :                 if (opt_abort)
        2434               0 :                         abort();
        2435                 :         }
        2436           27462 : }
        2437                 : #endif
        2438                 :
        2439                 : #ifdef MOZ_MEMORY_DARWIN
        2440                 : #define VM_COPY_MIN (pagesize << 5)
        2441                 : static inline void
        2442                 : pages_copy(void *dest, const void *src, size_t n)
        2443                 : {
        2444                 :
        2445                 :         assert((void *)((uintptr_t)dest & ~pagesize_mask) == dest);
        2446                 :         assert(n >= VM_COPY_MIN);
        2447                 :         assert((void *)((uintptr_t)src & ~pagesize_mask) == src);
        2448                 :
        2449                 :         vm_copy(mach_task_self(), (vm_address_t)src, (vm_size_t)n,
        2450                 :             (vm_address_t)dest);
        2451                 : }
        2452                 : #endif
        2453                 :
        2454                 : #ifdef MALLOC_VALIDATE
        2455                 : static inline malloc_rtree_t *
        2456           18669 : malloc_rtree_new(unsigned bits)
        2457                 : {
        2458                 :         malloc_rtree_t *ret;
        2459                 :         unsigned bits_per_level, height, i;
        2460                 :
        2461           37338 :         bits_per_level = ffs(pow2_ceil((MALLOC_RTREE_NODESIZE /
        2462           18669 :             sizeof(void *)))) - 1;
        2463           18669 :         height = bits / bits_per_level;
        2464           18669 :         if (height * bits_per_level != bits)
        2465               0 :                 height++;
        2466           18669 :         assert(height * bits_per_level >= bits);
        2467                 :
        2468           18669 :         ret = (malloc_rtree_t*)base_calloc(1, sizeof(malloc_rtree_t) +
        2469                 :             (sizeof(unsigned) * (height - 1)));
        2470           18669 :         if (ret == NULL)
        2471               0 :                 return (NULL);
        2472                 :
        2473           18669 :         malloc_spin_init(&ret->lock);
        2474           18669 :         ret->height = height;
        2475           18669 :         if (bits_per_level * height > bits)
        2476               0 :                 ret->level2bits[0] = bits % bits_per_level;
        2477                 :         else
        2478           18669 :                 ret->level2bits[0] = bits_per_level;
        2479           18669 :         for (i = 1; i < height; i++)
        2480               0 :                 ret->level2bits[i] = bits_per_level;
        2481                 :
        2482           18669 :         ret->root = (void**)base_calloc(1, sizeof(void *) << ret->level2bits[0]);
        2483           18669 :         if (ret->root == NULL) {
        2484                 :                 /*
        2485                 :                  * We leak the rtree here, since there's no generic base
        2486                 :                  * deallocation.
        2487                 :                  */
        2488               0 :                 return (NULL);
        2489                 :         }
        2490                 :
        2491           18669 :         return (ret);
        2492                 : }
        2493                 :
        2494                 : #define MALLOC_RTREE_GET_GENERATE(f)                                    \
        2495                 : /* The least significant bits of the key are ignored. */                \
        2496                 : static inline void *                                                    \
        2497                 : f(malloc_rtree_t *rtree, uintptr_t key)                                 \
        2498                 : {                                                                       \
        2499                 :         void *ret;                                                      \
        2500                 :         uintptr_t subkey;                                               \
        2501                 :         unsigned i, lshift, height, bits;                               \
        2502                 :         void **node, **child;                                           \
        2503                 :                                                                         \
        2504                 :         MALLOC_RTREE_LOCK(&rtree->lock);                         \
        2505                 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;\
        2506                 :             i < height - 1;                                          \
        2507                 :             i++, lshift += bits, node = child) {                        \
        2508                 :                 bits = rtree->level2bits[i];                         \
        2509                 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);       \
        2510                 :                 child = (void**)node[subkey];                           \
        2511                 :                 if (child == NULL) {                                    \
        2512                 :                         MALLOC_RTREE_UNLOCK(&rtree->lock);               \
        2513                 :                         return (NULL);                                  \
        2514                 :                 }                                                       \
        2515                 :         }                                                               \
        2516                 :                                                                         \
        2517                 :         /*                                                              \
        2518                 :          * node is a leaf, so it contains values rather than node       \
        2519                 :          * pointers.                                                    \
        2520                 :          */                                                             \
        2521                 :         bits = rtree->level2bits[i];                                 \
        2522                 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);               \
        2523                 :         ret = node[subkey];                                             \
        2524                 :         MALLOC_RTREE_UNLOCK(&rtree->lock);                               \
        2525                 :                                                                         \
        2526                 :         MALLOC_RTREE_GET_VALIDATE                                       \
        2527                 :         return (ret);                                                   \
        2528                 : }
        2529                 :
        2530                 : #ifdef MALLOC_DEBUG
        2531                 : #  define MALLOC_RTREE_LOCK(l)          malloc_spin_lock(l)
        2532                 : #  define MALLOC_RTREE_UNLOCK(l)        malloc_spin_unlock(l)
        2533                 : #  define MALLOC_RTREE_GET_VALIDATE
        2534               0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get_locked)
        2535                 : #  undef MALLOC_RTREE_LOCK
        2536                 : #  undef MALLOC_RTREE_UNLOCK
        2537                 : #  undef MALLOC_RTREE_GET_VALIDATE
        2538                 : #endif
        2539                 :
        2540                 : #define MALLOC_RTREE_LOCK(l)
        2541                 : #define MALLOC_RTREE_UNLOCK(l)
        2542                 : #ifdef MALLOC_DEBUG
        2543                 :    /*
        2544                 :     * Suppose that it were possible for a jemalloc-allocated chunk to be
        2545                 :     * munmap()ped, followed by a different allocator in another thread re-using
        2546                 :     * overlapping virtual memory, all without invalidating the cached rtree
        2547                 :     * value.  The result would be a false positive (the rtree would claim that
        2548                 :     * jemalloc owns memory that it had actually discarded).  I don't think this
        2549                 :     * scenario is possible, but the following assertion is a prudent sanity
        2550                 :     * check.
        2551                 :     */
        2552                 : #  define MALLOC_RTREE_GET_VALIDATE                                     \
        2553                 :         assert(malloc_rtree_get_locked(rtree, key) == ret);
        2554                 : #else
        2555                 : #  define MALLOC_RTREE_GET_VALIDATE
        2556                 : #endif
        2557               0 : MALLOC_RTREE_GET_GENERATE(malloc_rtree_get)
        2558                 : #undef MALLOC_RTREE_LOCK
        2559                 : #undef MALLOC_RTREE_UNLOCK
        2560                 : #undef MALLOC_RTREE_GET_VALIDATE
        2561                 :
        2562                 : static inline bool
        2563           33224 : malloc_rtree_set(malloc_rtree_t *rtree, uintptr_t key, void *val)
        2564                 : {
        2565                 :         uintptr_t subkey;
        2566                 :         unsigned i, lshift, height, bits;
        2567                 :         void **node, **child;
        2568                 :
        2569           33224 :         malloc_spin_lock(&rtree->lock);
        2570           66448 :         for (i = lshift = 0, height = rtree->height, node = rtree->root;
        2571           33224 :             i < height - 1;
        2572               0 :             i++, lshift += bits, node = child) {
        2573               0 :                 bits = rtree->level2bits[i];
        2574               0 :                 subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
        2575               0 :                 child = (void**)node[subkey];
        2576               0 :                 if (child == NULL) {
        2577               0 :                         child = (void**)base_calloc(1, sizeof(void *) <<
        2578               0 :                             rtree->level2bits[i+1]);
        2579               0 :                         if (child == NULL) {
        2580               0 :                                 malloc_spin_unlock(&rtree->lock);
        2581               0 :                                 return (true);
        2582                 :                         }
        2583               0 :                         node[subkey] = child;
        2584                 :                 }
        2585                 :         }
        2586                 :
        2587                 :         /* node is a leaf, so it contains values rather than node pointers. */
        2588           33224 :         bits = rtree->level2bits[i];
        2589           33224 :         subkey = (key << lshift) >> ((SIZEOF_PTR << 3) - bits);
        2590           33224 :         node[subkey] = val;
        2591           33224 :         malloc_spin_unlock(&rtree->lock);
        2592                 :
        2593           33224 :         return (false);
        2594                 : }
        2595                 : #endif
        2596                 :
        2597                 : #if defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)
        2598                 :
        2599                 : /* Allocate an aligned chunk while maintaining a 1:1 correspondence between
        2600                 :  * mmap and unmap calls.  This is important on Windows, but not elsewhere. */
        2601                 : static void *
        2602                 : chunk_alloc_mmap(size_t size, bool pagefile)
        2603                 : {
        2604                 :         void *ret;
        2605                 : #ifndef JEMALLOC_USES_MAP_ALIGN
        2606                 :         size_t offset;
        2607                 : #endif
        2608                 :         int pfd;
        2609                 :
        2610                 : #ifdef MALLOC_PAGEFILE
        2611                 :         if (opt_pagefile && pagefile) {
        2612                 :                 pfd = pagefile_init(size);
        2613                 :                 if (pfd == -1)
        2614                 :                         return (NULL);
        2615                 :         } else
        2616                 : #endif
        2617                 :                 pfd = -1;
        2618                 :
        2619                 : #ifdef JEMALLOC_USES_MAP_ALIGN
        2620                 :         ret = pages_map_align(size, pfd, chunksize);
        2621                 : #else
        2622                 :         ret = pages_map(NULL, size, pfd);
        2623                 :         if (ret == NULL)
        2624                 :                 goto RETURN;
        2625                 :
        2626                 :         offset = CHUNK_ADDR2OFFSET(ret);
        2627                 :         if (offset != 0) {
        2628                 :                 /* Deallocate, then try to allocate at (ret + size - offset). */
        2629                 :                 pages_unmap(ret, size);
        2630                 :                 ret = pages_map((void *)((uintptr_t)ret + size - offset), size,
        2631                 :                     pfd);
        2632                 :                 while (ret == NULL) {
        2633                 :                         /*
        2634                 :                          * Over-allocate in order to map a memory region that
        2635                 :                          * is definitely large enough.
        2636                 :                          */
        2637                 :                         ret = pages_map(NULL, size + chunksize, -1);
        2638                 :                         if (ret == NULL)
        2639                 :                                 goto RETURN;
        2640                 :                         /*
        2641                 :                          * Deallocate, then allocate the correct size, within
        2642                 :                          * the over-sized mapping.
        2643                 :                          */
        2644                 :                         offset = CHUNK_ADDR2OFFSET(ret);
        2645                 :                         pages_unmap(ret, size + chunksize);
        2646                 :                         if (offset == 0)
        2647                 :                                 ret = pages_map(ret, size, pfd);
        2648                 :                         else {
        2649                 :                                 ret = pages_map((void *)((uintptr_t)ret +
        2650                 :                                     chunksize - offset), size, pfd);
        2651                 :                         }
        2652                 :                         /*
        2653                 :                          * Failure here indicates a race with another thread, so
        2654                 :                          * try again.
        2655                 :                          */
        2656                 :                 }
        2657                 :         }
        2658                 : RETURN:
        2659                 : #endif
        2660                 : #ifdef MALLOC_PAGEFILE
        2661                 :         if (pfd != -1)
        2662                 :                 pagefile_close(pfd);
        2663                 : #endif
        2664                 : #ifdef MALLOC_STATS
        2665                 :         if (ret != NULL)
        2666                 :                 stats_chunks.nchunks += (size / chunksize);
        2667                 : #endif
        2668                 :         return (ret);
        2669                 : }
        2670                 :
        2671                 : #else /* ! (defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE)) */
        2672                 :
        2673                 : /*
        2674                 :  * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and
        2675                 :  * potentially avoid some system calls.
        2676                 :  */
        2677                 : #ifndef NO_TLS
        2678                 : static __thread bool    mmap_unaligned_tls __attribute__((tls_model("initial-exec")));
        2679                 : #define MMAP_UNALIGNED_GET()    mmap_unaligned_tls
        2680                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
        2681                 :         mmap_unaligned_tls = (v);                                       \
        2682                 : } while (0)
        2683                 : #else
        2684                 : #define NEEDS_PTHREAD_MMAP_UNALIGNED_TSD
        2685                 : static pthread_key_t    mmap_unaligned_tsd;
        2686                 : #define MMAP_UNALIGNED_GET()    ((bool)pthread_getspecific(mmap_unaligned_tsd))
        2687                 : #define MMAP_UNALIGNED_SET(v)   do {                                    \
        2688                 :         pthread_setspecific(mmap_unaligned_tsd, (void *)(v));           \
        2689                 : } while (0)
        2690                 : #endif
        2691                 :
        2692                 : /* chunk_alloc_mmap_slow and chunk_alloc_mmap were cherry-picked from upstream
        2693                 :  * jemalloc 2.2.3 to fix Mozilla bug 694896, enable jemalloc on Mac 10.7. */
        2694                 :
        2695                 : static void *
        2696            2961 : chunk_alloc_mmap_slow(size_t size, bool unaligned)
        2697                 : {
        2698                 :         void *ret;
        2699                 :         size_t offset;
        2700                 :
        2701                 :         /* Beware size_t wrap-around. */
        2702            2961 :         if (size + chunksize <= size)
        2703               0 :                 return (NULL);
        2704                 :
        2705            2961 :         ret = pages_map(NULL, size + chunksize, -1);
        2706            2961 :         if (ret == NULL)
        2707               0 :                 return (NULL);
        2708                 :
        2709                 :         /* Clean up unneeded leading/trailing space. */
        2710            2961 :         offset = CHUNK_ADDR2OFFSET(ret);
        2711            2961 :         if (offset != 0) {
        2712                 :                 /* Note that mmap() returned an unaligned mapping. */
        2713              10 :                 unaligned = true;
        2714                 :
        2715                 :                 /* Leading space. */
        2716              10 :                 pages_unmap(ret, chunksize - offset);
        2717                 :
        2718              10 :                 ret = (void *)((uintptr_t)ret +
        2719              10 :                     (chunksize - offset));
        2720                 :
        2721                 :                 /* Trailing space. */
        2722              10 :                 pages_unmap((void *)((uintptr_t)ret + size),
        2723                 :                     offset);
        2724                 :         } else {
        2725                 :                 /* Trailing space only. */
        2726            2951 :                 pages_unmap((void *)((uintptr_t)ret + size),
        2727                 :                     chunksize);
        2728                 :         }
        2729                 :
        2730                 :         /*
        2731                 :          * If mmap() returned an aligned mapping, reset mmap_unaligned so that
        2732                 :          * the next chunk_alloc_mmap() execution tries the fast allocation
        2733                 :          * method.
        2734                 :          */
        2735            2961 :         if (unaligned == false)
        2736            2951 :                 MMAP_UNALIGNED_SET(false);
        2737                 :
        2738            2961 :         return (ret);
        2739                 : }
        2740                 :
        2741                 : static void *
        2742           27428 : chunk_alloc_mmap(size_t size, bool pagefile)
        2743                 : {
        2744                 :         void *ret;
        2745                 :
        2746                 :         /*
        2747                 :          * Ideally, there would be a way to specify alignment to mmap() (like
        2748                 :          * NetBSD has), but in the absence of such a feature, we have to work
        2749                 :          * hard to efficiently create aligned mappings.  The reliable, but
        2750                 :          * slow method is to create a mapping that is over-sized, then trim the
        2751                 :          * excess.  However, that always results in at least one call to
        2752                 :          * pages_unmap().
        2753                 :          *
        2754                 :          * A more optimistic approach is to try mapping precisely the right
        2755                 :          * amount, then try to append another mapping if alignment is off.  In
        2756                 :          * practice, this works out well as long as the application is not
        2757                 :          * interleaving mappings via direct mmap() calls.  If we do run into a
        2758                 :          * situation where there is an interleaved mapping and we are unable to
        2759                 :          * extend an unaligned mapping, our best option is to switch to the
        2760                 :          * slow method until mmap() returns another aligned mapping.  This will
        2761                 :          * tend to leave a gap in the memory map that is too small to cause
        2762                 :          * later problems for the optimistic method.
        2763                 :          *
        2764                 :          * Another possible confounding factor is address space layout
        2765                 :          * randomization (ASLR), which causes mmap(2) to disregard the
        2766                 :          * requested address.  mmap_unaligned tracks whether the previous
        2767                 :          * chunk_alloc_mmap() execution received any unaligned or relocated
        2768                 :          * mappings, and if so, the current execution will immediately fall
        2769                 :          * back to the slow method.  However, we keep track of whether the fast
        2770                 :          * method would have succeeded, and if so, we make a note to try the
        2771                 :          * fast method next time.
        2772                 :          */
        2773                 :
        2774           27428 :         if (MMAP_UNALIGNED_GET() == false) {
        2775                 :                 size_t offset;
        2776                 :
        2777           24467 :                 ret = pages_map(NULL, size, -1);
        2778           24467 :                 if (ret == NULL)
        2779              18 :                         return (NULL);
        2780                 :
        2781           24449 :                 offset = CHUNK_ADDR2OFFSET(ret);
        2782           24449 :                 if (offset != 0) {
        2783           18677 :                         MMAP_UNALIGNED_SET(true);
        2784                 :                         /* Try to extend chunk boundary. */
        2785           18677 :                         if (pages_map((void *)((uintptr_t)ret + size),
        2786                 :                             chunksize - offset, -1) == NULL) {
        2787                 :                                 /*
        2788                 :                                  * Extension failed.  Clean up, then revert to
        2789                 :                                  * the reliable-but-expensive method.
        2790                 :                                  */
        2791               0 :                                 pages_unmap(ret, size);
        2792               0 :                                 ret = chunk_alloc_mmap_slow(size, true);
        2793                 :                         } else {
        2794                 :                                 /* Clean up unneeded leading space. */
        2795           18677 :                                 pages_unmap(ret, chunksize - offset);
        2796           18677 :                                 ret = (void *)((uintptr_t)ret + (chunksize -
        2797                 :                                     offset));
        2798                 :                         }
        2799                 :                 }
        2800                 :         } else
        2801            2961 :                 ret = chunk_alloc_mmap_slow(size, false);
        2802                 :
        2803           27410 :         return (ret);
        2804                 : }
        2805                 :
        2806                 : #endif /* defined(MOZ_MEMORY_WINDOWS) || defined(JEMALLOC_USES_MAP_ALIGN) || defined(MALLOC_PAGEFILE) */
        2807                 :
        2808                 : #ifdef MALLOC_PAGEFILE
        2809                 : static int
        2810                 : pagefile_init(size_t size)
        2811                 : {
        2812                 :         int ret;
        2813                 :         size_t i;
        2814                 :         char pagefile_path[PATH_MAX];
        2815                 :         char zbuf[MALLOC_PAGEFILE_WRITE_SIZE];
        2816                 :
        2817                 :         /*
        2818                 :          * Create a temporary file, then immediately unlink it so that it will
        2819                 :          * not persist.
        2820                 :          */
        2821                 :         strcpy(pagefile_path, pagefile_templ);
        2822                 :         ret = mkstemp(pagefile_path);
        2823                 :         if (ret == -1)
        2824                 :                 return (ret);
        2825                 :         if (unlink(pagefile_path)) {
        2826                 :                 char buf[STRERROR_BUF];
        2827                 :
        2828                 :                 strerror_r(errno, buf, sizeof(buf));
        2829                 :                 _malloc_message(_getprogname(), ": (malloc) Error in unlink(\"",
        2830                 :                     pagefile_path, "\"):");
        2831                 :                 _malloc_message(buf, "\n", "", "");
        2832                 :                 if (opt_abort)
        2833                 :                         abort();
        2834                 :         }
        2835                 :
        2836                 :         /*
        2837                 :          * Write sequential zeroes to the file in order to assure that disk
        2838                 :          * space is committed, with minimal fragmentation.  It would be
        2839                 :          * sufficient to write one zero per disk block, but that potentially
        2840                 :          * results in more system calls, for no real gain.
        2841                 :          */
        2842                 :         memset(zbuf, 0, sizeof(zbuf));
        2843                 :         for (i = 0; i < size; i += sizeof(zbuf)) {
        2844                 :                 if (write(ret, zbuf, sizeof(zbuf)) != sizeof(zbuf)) {
        2845                 :                         if (errno != ENOSPC) {
        2846                 :                                 char buf[STRERROR_BUF];
        2847                 :
        2848                 :                                 strerror_r(errno, buf, sizeof(buf));
        2849                 :                                 _malloc_message(_getprogname(),
        2850                 :                                     ": (malloc) Error in write(): ", buf, "\n");
        2851                 :                                 if (opt_abort)
        2852                 :                                         abort();
        2853                 :                         }
        2854                 :                         pagefile_close(ret);
        2855                 :                         return (-1);
        2856                 :                 }
        2857                 :         }
        2858                 :
        2859                 :         return (ret);
        2860                 : }
        2861                 :
        2862                 : static void
        2863                 : pagefile_close(int pfd)
        2864                 : {
        2865                 :
        2866                 :         if (close(pfd)) {
        2867                 :                 char buf[STRERROR_BUF];
        2868                 :
        2869                 :                 strerror_r(errno, buf, sizeof(buf));
        2870                 :                 _malloc_message(_getprogname(),
        2871                 :                     ": (malloc) Error in close(): ", buf, "\n");
        2872                 :                 if (opt_abort)
        2873                 :                         abort();
        2874                 :         }
        2875                 : }
        2876                 : #endif
        2877                 :
        2878                 : static void *
        2879           27428 : chunk_alloc(size_t size, bool zero, bool pagefile)
        2880                 : {
        2881                 :         void *ret;
        2882                 :
        2883           27428 :         assert(size != 0);
        2884           27428 :         assert((size & chunksize_mask) == 0);
        2885                 :
        2886           27428 :         ret = chunk_alloc_mmap(size, pagefile);
        2887           27428 :         if (ret != NULL) {
        2888           27410 :                 goto RETURN;
        2889                 :         }
        2890                 :
        2891                 :         /* All strategies for allocation failed. */
        2892              18 :         ret = NULL;
        2893                 : RETURN:
        2894                 : #ifdef MALLOC_STATS
        2895           27428 :         if (ret != NULL)
        2896           27410 :                 stats_chunks.curchunks += (size / chunksize);
        2897           27428 :         if (stats_chunks.curchunks > stats_chunks.highchunks)
        2898           23539 :                 stats_chunks.highchunks = stats_chunks.curchunks;
        2899                 : #endif
        2900                 :
        2901                 : #ifdef MALLOC_VALIDATE
        2902           27428 :         if (ret != NULL) {
        2903           27410 :                 if (malloc_rtree_set(chunk_rtree, (uintptr_t)ret, ret)) {
        2904               0 :                         chunk_dealloc(ret, size);
        2905               0 :                         return (NULL);
        2906                 :                 }
        2907                 :         }
        2908                 : #endif
        2909                 :
        2910           27428 :         assert(CHUNK_ADDR2BASE(ret) == ret);
        2911           27428 :         return (ret);
        2912                 : }
        2913                 :
        2914                 : static void
        2915            5814 : chunk_dealloc_mmap(void *chunk, size_t size)
        2916                 : {
        2917                 :
        2918            5814 :         pages_unmap(chunk, size);
        2919            5814 : }
        2920                 :
        2921                 : static void
        2922            5814 : chunk_dealloc(void *chunk, size_t size)
        2923                 : {
        2924                 :
        2925            5814 :         assert(chunk != NULL);
        2926            5814 :         assert(CHUNK_ADDR2BASE(chunk) == chunk);
        2927            5814 :         assert(size != 0);
        2928            5814 :         assert((size & chunksize_mask) == 0);
        2929                 :
        2930                 : #ifdef MALLOC_STATS
        2931            5814 :         stats_chunks.curchunks -= (size / chunksize);
        2932                 : #endif
        2933                 : #ifdef MALLOC_VALIDATE
        2934            5814 :         malloc_rtree_set(chunk_rtree, (uintptr_t)chunk, NULL);
        2935                 : #endif
        2936                 :
        2937            5814 :         chunk_dealloc_mmap(chunk, size);
        2938            5814 : }
        2939                 :
        2940                 : /*
        2941                 :  * End chunk management functions.
        2942                 :  */
        2943                 : /******************************************************************************/
        2944                 : /*
        2945                 :  * Begin arena.
        2946                 :  */
        2947                 :
        2948                 : /*
        2949                 :  * Choose an arena based on a per-thread value (fast-path code, calls slow-path
        2950                 :  * code if necessary).
        2951                 :  */
        2952                 : static inline arena_t *
        2953        46372702 : choose_arena(void)
        2954                 : {
        2955                 :         arena_t *ret;
        2956                 :
        2957                 :         /*
        2958                 :          * We can only use TLS if this is a PIC library, since for the static
        2959                 :          * library version, libc's malloc is used by TLS allocation, which
        2960                 :          * introduces a bootstrapping issue.
        2961                 :          */
        2962                 : #ifndef NO_TLS
        2963        46372702 :         if (isthreaded == false) {
        2964                 :             /* Avoid the overhead of TLS for single-threaded operation. */
        2965               0 :             return (arenas[0]);
        2966                 :         }
        2967                 :
        2968                 : #  ifdef MOZ_MEMORY_WINDOWS
        2969                 :         ret = (arena_t*)TlsGetValue(tlsIndex);
        2970                 : #  else
        2971        46372702 :         ret = arenas_map;
        2972                 : #  endif
        2973                 :
        2974        46372702 :         if (ret == NULL) {
        2975               5 :                 ret = choose_arena_hard();
        2976               5 :                 assert(ret != NULL);
        2977                 :         }
        2978                 : #else
        2979                 :         if (isthreaded && narenas > 1) {
        2980                 :                 unsigned long ind;
        2981                 :
        2982                 :                 /*
        2983                 :                  * Hash _pthread_self() to one of the arenas.  There is a prime
        2984                 :                  * number of arenas, so this has a reasonable chance of
        2985                 :                  * working.  Even so, the hashing can be easily thwarted by
        2986                 :                  * inconvenient _pthread_self() values.  Without specific
        2987                 :                  * knowledge of how _pthread_self() calculates values, we can't
        2988                 :                  * easily do much better than this.
        2989                 :                  */
        2990                 :                 ind = (unsigned long) _pthread_self() % narenas;
        2991                 :
        2992                 :                 /*
        2993                 :                  * Optimistially assume that arenas[ind] has been initialized.
        2994                 :                  * At worst, we find out that some other thread has already
        2995                 :                  * done so, after acquiring the lock in preparation.  Note that
        2996                 :                  * this lazy locking also has the effect of lazily forcing
        2997                 :                  * cache coherency; without the lock acquisition, there's no
        2998                 :                  * guarantee that modification of arenas[ind] by another thread
        2999                 :                  * would be seen on this CPU for an arbitrary amount of time.
        3000                 :                  *
        3001                 :                  * In general, this approach to modifying a synchronized value
        3002                 :                  * isn't a good idea, but in this case we only ever modify the
        3003                 :                  * value once, so things work out well.
        3004                 :                  */
        3005                 :                 ret = arenas[ind];
        3006                 :                 if (ret == NULL) {
        3007                 :                         /*
        3008                 :                          * Avoid races with another thread that may have already
        3009                 :                          * initialized arenas[ind].
        3010                 :                          */
        3011                 :                         malloc_spin_lock(&arenas_lock);
        3012                 :                         if (arenas[ind] == NULL)
        3013                 :                                 ret = arenas_extend((unsigned)ind);
        3014                 :                         else
        3015                 :                                 ret = arenas[ind];
        3016                 :                         malloc_spin_unlock(&arenas_lock);
        3017                 :                 }
        3018                 :         } else
        3019                 :                 ret = arenas[0];
        3020                 : #endif
        3021                 :
        3022        46372702 :         assert(ret != NULL);
        3023        46372702 :         return (ret);
        3024                 : }
        3025                 :
        3026                 : #ifndef NO_TLS
        3027                 : /*
        3028                 :  * Choose an arena based on a per-thread value (slow-path code only, called
        3029                 :  * only by choose_arena()).
        3030                 :  */
        3031                 : static arena_t *
        3032               5 : choose_arena_hard(void)
        3033                 : {
        3034                 :         arena_t *ret;
        3035                 :
        3036               5 :         assert(isthreaded);
        3037                 :
        3038                 : #ifdef MALLOC_BALANCE
        3039                 :         /* Seed the PRNG used for arena load balancing. */
        3040                 :         SPRN(balance, (uint32_t)(uintptr_t)(_pthread_self()));
        3041                 : #endif
        3042                 :
        3043               5 :         if (narenas > 1) {
        3044                 : #ifdef MALLOC_BALANCE
        3045                 :                 unsigned ind;
        3046                 :
        3047                 :                 ind = PRN(balance, narenas_2pow);
        3048                 :                 if ((ret = arenas[ind]) == NULL) {
        3049                 :                         malloc_spin_lock(&arenas_lock);
        3050                 :                         if ((ret = arenas[ind]) == NULL)
        3051                 :                                 ret = arenas_extend(ind);
        3052                 :                         malloc_spin_unlock(&arenas_lock);
        3053                 :                 }
        3054                 : #else
        3055               0 :                 malloc_spin_lock(&arenas_lock);
        3056               0 :                 if ((ret = arenas[next_arena]) == NULL)
        3057               0 :                         ret = arenas_extend(next_arena);
        3058               0 :                 next_arena = (next_arena + 1) % narenas;
        3059               0 :                 malloc_spin_unlock(&arenas_lock);
        3060                 : #endif
        3061                 :         } else
        3062               5 :                 ret = arenas[0];
        3063                 :
        3064                 : #ifdef MOZ_MEMORY_WINDOWS
        3065                 :         TlsSetValue(tlsIndex, ret);
        3066                 : #else
        3067               5 :         arenas_map = ret;
        3068                 : #endif
        3069                 :
        3070               5 :         return (ret);
        3071                 : }
        3072                 : #endif
        3073                 :
        3074                 : static inline int
        3075          194091 : arena_chunk_comp(arena_chunk_t *a, arena_chunk_t *b)
        3076                 : {
        3077          194091 :         uintptr_t a_chunk = (uintptr_t)a;
        3078          194091 :         uintptr_t b_chunk = (uintptr_t)b;
        3079                 :
        3080          194091 :         assert(a != NULL);
        3081          194091 :         assert(b != NULL);
        3082                 :
        3083          194091 :         return ((a_chunk > b_chunk) - (a_chunk < b_chunk));
        3084                 : }
        3085                 :
        3086                 : /* Wrap red-black tree macros in functions. */
        3087          313285 : rb_wrap(static, arena_chunk_tree_dirty_, arena_chunk_tree_t,
        3088          313285 :     arena_chunk_t, link_dirty, arena_chunk_comp)
        3089                 :
        3090                 : static inline int
        3091         1479607 : arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
        3092                 : {
        3093         1479607 :         uintptr_t a_mapelm = (uintptr_t)a;
        3094         1479607 :         uintptr_t b_mapelm = (uintptr_t)b;
        3095                 :
        3096         1479607 :         assert(a != NULL);
        3097         1479607 :         assert(b != NULL);
        3098                 :
        3099         1479607 :         return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm));
        3100                 : }
        3101                 :
        3102                 : /* Wrap red-black tree macros in functions. */
        3103         4897235 : rb_wrap(static, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, link,
        3104         4897235 :     arena_run_comp)
        3105                 :
        3106                 : static inline int
        3107        27108048 : arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b)
        3108                 : {
        3109                 :         int ret;
        3110        27108048 :         size_t a_size = a->bits & ~pagesize_mask;
        3111        27108048 :         size_t b_size = b->bits & ~pagesize_mask;
        3112                 :
        3113        27108048 :         ret = (a_size > b_size) - (a_size < b_size);
        3114        27108048 :         if (ret == 0) {
        3115                 :                 uintptr_t a_mapelm, b_mapelm;
        3116                 :
        3117         7525083 :                 if ((a->bits & CHUNK_MAP_KEY) == 0)
        3118         7407268 :                         a_mapelm = (uintptr_t)a;
        3119                 :                 else {
        3120                 :                         /*
        3121                 :                          * Treat keys as though they are lower than anything
        3122                 :                          * else.
        3123                 :                          */
        3124          117815 :                         a_mapelm = 0;
        3125                 :                 }
        3126         7525083 :                 b_mapelm = (uintptr_t)b;
        3127                 :
        3128         7525083 :                 ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm);
        3129                 :         }
        3130                 :
        3131        27108048 :         return (ret);
        3132                 : }
        3133                 :
        3134                 : /* Wrap red-black tree macros in functions. */
        3135        17699161 : rb_wrap(static, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, link,
        3136        17699161 :     arena_avail_comp)
        3137                 :
        3138                 : static inline void *
        3139        45735071 : arena_run_reg_alloc(arena_run_t *run, arena_bin_t *bin)
        3140                 : {
        3141                 :         void *ret;
        3142                 :         unsigned i, mask, bit, regind;
        3143                 :
        3144        45735071 :         assert(run->magic == ARENA_RUN_MAGIC);
        3145        45735071 :         assert(run->regs_minelm < bin->regs_mask_nelms);
        3146                 :
        3147                 :         /*
        3148                 :          * Move the first check outside the loop, so that run->regs_minelm can
        3149                 :          * be updated unconditionally, without the possibility of updating it
        3150                 :          * multiple times.
        3151                 :          */
        3152        45735071 :         i = run->regs_minelm;
        3153        45735071 :         mask = run->regs_mask[i];
        3154        45735071 :         if (mask != 0) {
        3155                 :                 /* Usable allocation found. */
        3156        44908552 :                 bit = ffs((int)mask) - 1;
        3157                 :
        3158        44908552 :                 regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
        3159        44908552 :                 assert(regind < bin->nregs);
        3160        89817104 :                 ret = (void *)(((uintptr_t)run) + bin->reg0_offset
        3161        44908552 :                     + (bin->reg_size * regind));
        3162                 :
        3163                 :                 /* Clear bit. */
        3164        44908552 :                 mask ^= (1U << bit);
        3165        44908552 :                 run->regs_mask[i] = mask;
        3166                 :
        3167        44908552 :                 return (ret);
        3168                 :         }
        3169                 :
        3170          979239 :         for (i++; i < bin->regs_mask_nelms; i++) {
        3171          979239 :                 mask = run->regs_mask[i];
        3172          979239 :                 if (mask != 0) {
        3173                 :                         /* Usable allocation found. */
        3174          826519 :                         bit = ffs((int)mask) - 1;
        3175                 :
        3176          826519 :                         regind = ((i << (SIZEOF_INT_2POW + 3)) + bit);
        3177          826519 :                         assert(regind < bin->nregs);
        3178         1653038 :                         ret = (void *)(((uintptr_t)run) + bin->reg0_offset
        3179          826519 :                             + (bin->reg_size * regind));
        3180                 :
        3181                 :                         /* Clear bit. */
        3182          826519 :                         mask ^= (1U << bit);
        3183          826519 :                         run->regs_mask[i] = mask;
        3184                 :
        3185                 :                         /*
        3186                 :                          * Make a note that nothing before this element
        3187                 :                          * contains a free region.
        3188                 :                          */
        3189          826519 :                         run->regs_minelm = i; /* Low payoff: + (mask == 0); */
        3190                 :
        3191          826519 :                         return (ret);
        3192                 :                 }
        3193                 :         }
        3194                 :         /* Not reached. */
        3195               0 :         assert(0);
        3196                 :         return (NULL);
        3197                 : }
        3198                 :
        3199                 : static inline void
        3200        43659702 : arena_run_reg_dalloc(arena_run_t *run, arena_bin_t *bin, void *ptr, size_t size)
        3201                 : {
        3202                 :         /*
        3203                 :          * To divide by a number D that is not a power of two we multiply
        3204                 :          * by (2^21 / D) and then right shift by 21 positions.
        3205                 :          *
        3206                 :          *   X / D
        3207                 :          *
        3208                 :          * becomes
        3209                 :          *
        3210                 :          *   (X * size_invs[(D >> QUANTUM_2POW_MIN) - 3]) >> SIZE_INV_SHIFT
        3211                 :          */
        3212                 : #define SIZE_INV_SHIFT 21
        3213                 : #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s << QUANTUM_2POW_MIN)) + 1)
        3214                 :         static const unsigned size_invs[] = {
        3215                 :             SIZE_INV(3),
        3216                 :             SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
        3217                 :             SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
        3218                 :             SIZE_INV(12),SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
        3219                 :             SIZE_INV(16),SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
        3220                 :             SIZE_INV(20),SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
        3221                 :             SIZE_INV(24),SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
        3222                 :             SIZE_INV(28),SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
        3223                 : #if (QUANTUM_2POW_MIN < 4)
        3224                 :             ,
        3225                 :             SIZE_INV(32), SIZE_INV(33), SIZE_INV(34), SIZE_INV(35),
        3226                 :             SIZE_INV(36), SIZE_INV(37), SIZE_INV(38), SIZE_INV(39),
        3227                 :             SIZE_INV(40), SIZE_INV(41), SIZE_INV(42), SIZE_INV(43),
        3228                 :             SIZE_INV(44), SIZE_INV(45), SIZE_INV(46), SIZE_INV(47),
        3229                 :             SIZE_INV(48), SIZE_INV(49), SIZE_INV(50), SIZE_INV(51),
        3230                 :             SIZE_INV(52), SIZE_INV(53), SIZE_INV(54), SIZE_INV(55),
        3231                 :             SIZE_INV(56), SIZE_INV(57), SIZE_INV(58), SIZE_INV(59),
        3232                 :             SIZE_INV(60), SIZE_INV(61), SIZE_INV(62), SIZE_INV(63)
        3233                 : #endif
        3234                 :         };
        3235                 :         unsigned diff, regind, elm, bit;
        3236                 :
        3237        43659702 :         assert(run->magic == ARENA_RUN_MAGIC);
        3238                 :         assert(((sizeof(size_invs)) / sizeof(unsigned)) + 3
        3239                 :             >= (SMALL_MAX_DEFAULT >> QUANTUM_2POW_MIN));
        3240                 :
        3241                 :         /*
        3242                 :          * Avoid doing division with a variable divisor if possible.  Using
        3243                 :          * actual division here can reduce allocator throughput by over 20%!
        3244                 :          */
        3245        43659702 :         diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - bin->reg0_offset);
        3246        43659702 :         if ((size & (size - 1)) == 0) {
        3247                 :                 /*
        3248                 :                  * log2_table allows fast division of a power of two in the
        3249                 :                  * [1..128] range.
        3250                 :                  *
        3251                 :                  * (x / divisor) becomes (x >> log2_table[divisor - 1]).
        3252                 :                  */
        3253                 :                 static const unsigned char log2_table[] = {
        3254                 :                     0, 1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4,
        3255                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5,
        3256                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        3257                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6,
        3258                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        3259                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        3260                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        3261                 :                     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
        3262                 :                 };
        3263                 :
        3264        34733586 :                 if (size <= 128)
        3265        28967596 :                         regind = (diff >> log2_table[size - 1]);
        3266         5765990 :                 else if (size <= 32768)
        3267         5765990 :                         regind = diff >> (8 + log2_table[(size >> 8) - 1]);
        3268                 :                 else {
        3269                 :                         /*
        3270                 :                          * The run size is too large for us to use the lookup
        3271                 :                          * table.  Use real division.
        3272                 :                          */
        3273               0 :                         regind = diff / size;
        3274                 :                 }
        3275         8926116 :         } else if (size <= ((sizeof(size_invs) / sizeof(unsigned))
        3276                 :             << QUANTUM_2POW_MIN) + 2) {
        3277         8896621 :                 regind = size_invs[(size >> QUANTUM_2POW_MIN) - 3] * diff;
        3278         8896621 :                 regind >>= SIZE_INV_SHIFT;
        3279                 :         } else {
        3280                 :                 /*
        3281                 :                  * size_invs isn't large enough to handle this size class, so
        3282                 :                  * calculate regind using actual division.  This only happens
        3283                 :                  * if the user increases small_max via the 'S' runtime
        3284                 :                  * configuration option.
        3285                 :                  */
        3286           29495 :                 regind = diff / size;
        3287                 :         };
        3288        43659702 :         assert(diff == regind * size);
        3289        43659702 :         assert(regind < bin->nregs);
        3290                 :
        3291        43659702 :         elm = regind >> (SIZEOF_INT_2POW + 3);
        3292        43659702 :         if (elm < run->regs_minelm)
        3293          624208 :                 run->regs_minelm = elm;
        3294        43659702 :         bit = regind - (elm << (SIZEOF_INT_2POW + 3));
        3295        43659702 :         assert((run->regs_mask[elm] & (1U << bit)) == 0);
        3296        43659702 :         run->regs_mask[elm] |= (1U << bit);
        3297                 : #undef SIZE_INV
        3298                 : #undef SIZE_INV_SHIFT
        3299        43659702 : }
        3300                 :
        3301                 : static void
        3302         3698730 : arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
        3303                 :     bool zero)
        3304                 : {
        3305                 :         arena_chunk_t *chunk;
        3306                 :         size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i;
        3307                 :
        3308         3698730 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
        3309         3698730 :         old_ndirty = chunk->ndirty;
        3310         3698730 :         run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk)
        3311                 :             >> pagesize_2pow);
        3312         3698730 :         total_pages = (chunk->map[run_ind].bits & ~pagesize_mask) >>
        3313                 :             pagesize_2pow;
        3314         3698730 :         need_pages = (size >> pagesize_2pow);
        3315         3698730 :         assert(need_pages > 0);
        3316         3698730 :         assert(need_pages <= total_pages);
        3317         3698730 :         rem_pages = total_pages - need_pages;
        3318                 :
        3319         3698730 :         arena_avail_tree_remove(&arena->runs_avail, &chunk->map[run_ind]);
        3320                 :
        3321                 :         /* Keep track of trailing unused pages for later use. */
        3322         3698730 :         if (rem_pages > 0) {
        3323         7185680 :                 chunk->map[run_ind+need_pages].bits = (rem_pages <<
        3324         3592840 :                     pagesize_2pow) | (chunk->map[run_ind+need_pages].bits &
        3325                 :                     pagesize_mask);
        3326         7185680 :                 chunk->map[run_ind+total_pages-1].bits = (rem_pages <<
        3327         3592840 :                     pagesize_2pow) | (chunk->map[run_ind+total_pages-1].bits &
        3328                 :                     pagesize_mask);
        3329         3592840 :                 arena_avail_tree_insert(&arena->runs_avail,
        3330         3592840 :                     &chunk->map[run_ind+need_pages]);
        3331                 :         }
        3332                 :
        3333        21780038 :         for (i = 0; i < need_pages; i++) {
        3334                 : #if defined(MALLOC_DECOMMIT) || defined(MALLOC_STATS) || defined(MALLOC_DOUBLE_PURGE)
        3335                 :                 /*
        3336                 :                  * Commit decommitted pages if necessary.  If a decommitted
        3337                 :                  * page is encountered, commit all needed adjacent decommitted
        3338                 :                  * pages in one operation, in order to reduce system call
        3339                 :                  * overhead.
        3340                 :                  */
        3341        18081308 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED) {
        3342                 :                         size_t j;
        3343                 :
        3344                 :                         /*
        3345                 :                          * Advance i+j to just past the index of the last page
        3346                 :                          * to commit.  Clear CHUNK_MAP_DECOMMITTED and
        3347                 :                          * CHUNK_MAP_MADVISED along the way.
        3348                 :                          */
        3349        11216558 :                         for (j = 0; i + j < need_pages && (chunk->map[run_ind +
        3350         9221752 :                             i + j].bits & CHUNK_MAP_MADVISED_OR_DECOMMITTED); j++) {
        3351                 :                                 /* DECOMMITTED and MADVISED are mutually exclusive. */
        3352         4610876 :                                 assert(!(chunk->map[run_ind + i + j].bits & CHUNK_MAP_DECOMMITTED &&
        3353                 :                                          chunk->map[run_ind + i + j].bits & CHUNK_MAP_MADVISED));
        3354                 :
        3355         4610876 :                                 chunk->map[run_ind + i + j].bits &=
        3356                 :                                     ~CHUNK_MAP_MADVISED_OR_DECOMMITTED;
        3357                 :                         }
        3358                 :
        3359                 : #  ifdef MALLOC_DECOMMIT
        3360                 :                         pages_commit((void *)((uintptr_t)chunk + ((run_ind + i)
        3361                 :                             << pagesize_2pow)), (j << pagesize_2pow));
        3362                 : #    ifdef MALLOC_STATS
        3363                 :                         arena->stats.ncommit++;
        3364                 : #    endif
        3365                 : #  endif
        3366                 :
        3367                 : #  ifdef MALLOC_STATS
        3368          997403 :                         arena->stats.committed += j;
        3369                 : #  endif
        3370                 :
        3371                 : #  ifndef MALLOC_DECOMMIT
        3372                 :                 }
        3373                 : #  else
        3374                 :                 } else /* No need to zero since commit zeros. */
        3375                 : #  endif
        3376                 :
        3377                 : #endif
        3378                 :
        3379                 :                 /* Zero if necessary. */
        3380        18081308 :                 if (zero) {
        3381          790138 :                         if ((chunk->map[run_ind + i].bits & CHUNK_MAP_ZEROED)
        3382                 :                             == 0) {
        3383                 :                                 VALGRIND_MALLOCLIKE_BLOCK((void *)((uintptr_t)
        3384                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
        3385                 :                                     pagesize, 0, false);
        3386          669225 :                                 memset((void *)((uintptr_t)chunk + ((run_ind
        3387          669225 :                                     + i) << pagesize_2pow)), 0, pagesize);
        3388                 :                                 VALGRIND_FREELIKE_BLOCK((void *)((uintptr_t)
        3389                 :                                     chunk + ((run_ind + i) << pagesize_2pow)),
        3390                 :                                     0);
        3391                 :                                 /* CHUNK_MAP_ZEROED is cleared below. */
        3392                 :                         }
        3393                 :                 }
        3394                 :
        3395                 :                 /* Update dirty page accounting. */
        3396        18081308 :                 if (chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY) {
        3397        13470432 :                         chunk->ndirty--;
        3398        13470432 :                         arena->ndirty--;
        3399                 :                         /* CHUNK_MAP_DIRTY is cleared below. */
        3400                 :                 }
        3401                 :
        3402                 :                 /* Initialize the chunk map. */
        3403        18081308 :                 if (large) {
        3404         4539996 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_LARGE
        3405                 :                             | CHUNK_MAP_ALLOCATED;
        3406                 :                 } else {
        3407        27082624 :                         chunk->map[run_ind + i].bits = (size_t)run
        3408        13541312 :                             | CHUNK_MAP_ALLOCATED;
        3409                 :                 }
        3410                 :         }
        3411                 :
        3412                 :         /*
        3413                 :          * Set the run size only in the first element for large runs.  This is
        3414                 :          * primarily a debugging aid, since the lack of size info for trailing
        3415                 :          * pages only matters if the application tries to operate on an
        3416                 :          * interior pointer.
        3417                 :          */
        3418         3698730 :         if (large)
        3419          646749 :                 chunk->map[run_ind].bits |= size;
        3420                 :
        3421         3698730 :         if (chunk->ndirty == 0 && old_ndirty > 0)
        3422          133997 :                 arena_chunk_tree_dirty_remove(&arena->chunks_dirty, chunk);
        3423         3698730 : }
        3424                 :
        3425                 : static void
        3426           22874 : arena_chunk_init(arena_t *arena, arena_chunk_t *chunk)
        3427                 : {
        3428                 :         arena_run_t *run;
        3429                 :         size_t i;
        3430                 :
        3431                 :         VALGRIND_MALLOCLIKE_BLOCK(chunk, (arena_chunk_header_npages <<
        3432                 :             pagesize_2pow), 0, false);
        3433                 : #ifdef MALLOC_STATS
        3434           22874 :         arena->stats.mapped += chunksize;
        3435                 : #endif
        3436                 :
        3437           22874 :         chunk->arena = arena;
        3438                 :
        3439                 :         /*
        3440                 :          * Claim that no pages are in use, since the header is merely overhead.
        3441                 :          */
        3442           22874 :         chunk->ndirty = 0;
        3443                 :
        3444                 :         /* Initialize the map to contain one maximal free untouched run. */
        3445           22874 :         run = (arena_run_t *)((uintptr_t)chunk + (arena_chunk_header_npages <<
        3446                 :             pagesize_2pow));
        3447           45748 :         for (i = 0; i < arena_chunk_header_npages; i++)
        3448           22874 :                 chunk->map[i].bits = 0;
        3449           22874 :         chunk->map[i].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
        3450         5809996 :         for (i++; i < chunk_npages-1; i++) {
        3451         5787122 :                 chunk->map[i].bits = CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
        3452                 :         }
        3453           22874 :         chunk->map[chunk_npages-1].bits = arena_maxclass | CHUNK_MAP_DECOMMITTED | CHUNK_MAP_ZEROED;
        3454                 :
        3455                 : #ifdef MALLOC_DECOMMIT
        3456                 :         /*
        3457                 :          * Start out decommitted, in order to force a closer correspondence
        3458                 :          * between dirty pages and committed untouched pages.
        3459                 :          */
        3460                 :         pages_decommit(run, arena_maxclass);
        3461                 : #  ifdef MALLOC_STATS
        3462                 :         arena->stats.ndecommit++;
        3463                 :         arena->stats.decommitted += (chunk_npages - arena_chunk_header_npages);
        3464                 : #  endif
        3465                 : #endif
        3466                 : #ifdef MALLOC_STATS
        3467           22874 :         arena->stats.committed += arena_chunk_header_npages;
        3468                 : #endif
        3469                 :
        3470                 :         /* Insert the run into the runs_avail tree. */
        3471           22874 :         arena_avail_tree_insert(&arena->runs_avail,
        3472           22874 :             &chunk->map[arena_chunk_header_npages]);
        3473                 :
        3474                 : #ifdef MALLOC_DOUBLE_PURGE
        3475                 :         LinkedList_Init(&chunk->chunks_madvised_elem);
        3476                 : #endif
        3477           22874 : }
        3478                 :
        3479                 : static void
        3480           30235 : arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
        3481                 : {
        3482                 :
        3483           30235 :         if (arena->spare != NULL) {
        3484            1278 :                 if (arena->spare->ndirty > 0) {
        3485            2392 :                         arena_chunk_tree_dirty_remove(
        3486            1196 :                             &chunk->arena->chunks_dirty, arena->spare);
        3487            1196 :                         arena->ndirty -= arena->spare->ndirty;
        3488                 : #ifdef MALLOC_STATS
        3489            1196 :                         arena->stats.committed -= arena->spare->ndirty;
        3490                 : #endif
        3491                 :                 }
        3492                 :
        3493                 : #ifdef MALLOC_DOUBLE_PURGE
        3494                 :                 /* This is safe to do even if arena->spare is not in the list. */
        3495                 :                 LinkedList_Remove(&arena->spare->chunks_madvised_elem);
        3496                 : #endif
        3497                 :
        3498                 :                 VALGRIND_FREELIKE_BLOCK(arena->spare, 0);
        3499            1278 :                 chunk_dealloc((void *)arena->spare, chunksize);
        3500                 : #ifdef MALLOC_STATS
        3501            1278 :                 arena->stats.mapped -= chunksize;
        3502            1278 :                 arena->stats.committed -= arena_chunk_header_npages;
        3503                 : #endif
        3504                 :         }
        3505                 :
        3506                 :         /*
        3507                 :          * Remove run from runs_avail, so that the arena does not use it.
        3508                 :          * Dirty page flushing only uses the chunks_dirty tree, so leaving this
        3509                 :          * chunk in the chunks_* trees is sufficient for that purpose.
        3510                 :          */
        3511           30235 :         arena_avail_tree_remove(&arena->runs_avail,
        3512           30235 :             &chunk->map[arena_chunk_header_npages]);
        3513                 :
        3514           30235 :         arena->spare = chunk;
        3515           30235 : }
        3516                 :
        3517                 : static arena_run_t *
        3518         3689612 : arena_run_alloc(arena_t *arena, arena_bin_t *bin, size_t size, bool large,
        3519                 :     bool zero)
        3520                 : {
        3521                 :         arena_run_t *run;
        3522                 :         arena_chunk_map_t *mapelm, key;
        3523                 :
        3524         3689612 :         assert(size <= arena_maxclass);
        3525         3689612 :         assert((size & pagesize_mask) == 0);
        3526                 :
        3527                 :         /* Search the arena's chunks for the lowest best fit. */
        3528         3689612 :         key.bits = size | CHUNK_MAP_KEY;
        3529         3689612 :         mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key);
        3530         3689612 :         if (mapelm != NULL) {
        3531         3640705 :                 arena_chunk_t *chunk =
        3532         3640705 :                     (arena_chunk_t*)CHUNK_ADDR2BASE(mapelm);
        3533         7281410 :                 size_t pageind = ((uintptr_t)mapelm -
        3534         3640705 :                     (uintptr_t)chunk->map) /
        3535                 :                     sizeof(arena_chunk_map_t);
        3536                 :
        3537         3640705 :                 run = (arena_run_t *)((uintptr_t)chunk + (pageind
        3538         3640705 :                     << pagesize_2pow));
        3539         3640705 :                 arena_run_split(arena, run, size, large, zero);
        3540         3640705 :                 return (run);
        3541                 :         }
        3542                 :
        3543           48907 :         if (arena->spare != NULL) {
        3544                 :                 /* Use the spare. */
        3545           26033 :                 arena_chunk_t *chunk = arena->spare;
        3546           26033 :                 arena->spare = NULL;
        3547           26033 :                 run = (arena_run_t *)((uintptr_t)chunk +
        3548           26033 :                     (arena_chunk_header_npages << pagesize_2pow));
        3549                 :                 /* Insert the run into the runs_avail tree. */
        3550           26033 :                 arena_avail_tree_insert(&arena->runs_avail,
        3551           26033 :                     &chunk->map[arena_chunk_header_npages]);
        3552           26033 :                 arena_run_split(arena, run, size, large, zero);
        3553           26033 :                 return (run);
        3554                 :         }
        3555                 :
        3556                 :         /*
        3557                 :          * No usable runs.  Create a new chunk from which to allocate
        3558                 :          * the run.
        3559                 :          */
        3560                 :         {
        3561           22874 :                 arena_chunk_t *chunk = (arena_chunk_t *)
        3562           22874 :                     chunk_alloc(chunksize, true, true);
        3563           22874 :                 if (chunk == NULL)
        3564               0 :                         return (NULL);
        3565                 :
        3566           22874 :                 arena_chunk_init(arena, chunk);
        3567           22874 :                 run = (arena_run_t *)((uintptr_t)chunk +
        3568           22874 :                     (arena_chunk_header_npages << pagesize_2pow));
        3569                 :         }
        3570                 :         /* Update page map. */
        3571           22874 :         arena_run_split(arena, run, size, large, zero);
        3572           22874 :         return (run);
        3573                 : }
        3574                 :
        3575                 : static void
        3576             165 : arena_purge(arena_t *arena)
        3577                 : {
        3578                 :         arena_chunk_t *chunk;
        3579                 :         size_t i, npages;
        3580                 : #ifdef MALLOC_DEBUG
        3581             165 :         size_t ndirty = 0;
        3582            1896 :         rb_foreach_begin(arena_chunk_t, link_dirty, &arena->chunks_dirty,
        3583                 :             chunk) {
        3584            1566 :                 ndirty += chunk->ndirty;
        3585            1566 :         } rb_foreach_end(arena_chunk_t, link_dirty, &arena->chunks_dirty, chunk)
        3586             165 :         assert(ndirty == arena->ndirty);
        3587                 : #endif
        3588             165 :         assert(arena->ndirty > opt_dirty_max);
        3589                 :
        3590                 : #ifdef MALLOC_STATS
        3591             165 :         arena->stats.npurge++;
        3592                 : #endif
        3593                 :
        3594                 :         /*
        3595                 :          * Iterate downward through chunks until enough dirty memory has been
        3596                 :          * purged.  Terminate as soon as possible in order to minimize the
        3597                 :          * number of system calls, even if a chunk has only been partially
        3598                 :          * purged.
        3599                 :          */
        3600            1244 :         while (arena->ndirty > (opt_dirty_max >> 1)) {
        3601                 : #ifdef MALLOC_DOUBLE_PURGE
        3602                 :                 bool madvised = false;
        3603                 : #endif
        3604             914 :                 chunk = arena_chunk_tree_dirty_last(&arena->chunks_dirty);
        3605             914 :                 assert(chunk != NULL);
        3606                 :
        3607           73094 :                 for (i = chunk_npages - 1; chunk->ndirty > 0; i--) {
        3608           72345 :                         assert(i >= arena_chunk_header_npages);
        3609                 :
        3610           72345 :                         if (chunk->map[i].bits & CHUNK_MAP_DIRTY) {
        3611                 : #ifdef MALLOC_DECOMMIT
        3612                 :                                 const size_t free_operation = CHUNK_MAP_DECOMMITTED;
        3613                 : #else
        3614            1770 :                                 const size_t free_operation = CHUNK_MAP_MADVISED;
        3615                 : #endif
        3616            1770 :                                 assert((chunk->map[i].bits &
        3617                 :                                         CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
        3618            1770 :                                 chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
        3619                 :                                 /* Find adjacent dirty run(s). */
        3620           97438 :                                 for (npages = 1;
        3621          190931 :                                      i > arena_chunk_header_npages &&
        3622           95263 :                                        (chunk->map[i - 1].bits & CHUNK_MAP_DIRTY);
        3623           93898 :                                      npages++) {
        3624           93898 :                                         i--;
        3625           93898 :                                         assert((chunk->map[i].bits &
        3626                 :                                                 CHUNK_MAP_MADVISED_OR_DECOMMITTED) == 0);
        3627           93898 :                                         chunk->map[i].bits ^= free_operation | CHUNK_MAP_DIRTY;
        3628                 :                                 }
        3629            1770 :                                 chunk->ndirty -= npages;
        3630            1770 :                                 arena->ndirty -= npages;
        3631                 :
        3632                 : #ifdef MALLOC_DECOMMIT
        3633                 :                                 pages_decommit((void *)((uintptr_t)
        3634                 :                                     chunk + (i << pagesize_2pow)),
        3635                 :                                     (npages << pagesize_2pow));
        3636                 : #  ifdef MALLOC_STATS
        3637                 :                                 arena->stats.ndecommit++;
        3638                 :                                 arena->stats.decommitted += npages;
        3639                 : #  endif
        3640                 : #endif
        3641                 : #ifdef MALLOC_STATS
        3642            1770 :                                 arena->stats.committed -= npages;
        3643                 : #endif
        3644                 :
        3645                 : #ifndef MALLOC_DECOMMIT
        3646            1770 :                                 madvise((void *)((uintptr_t)chunk + (i <<
        3647                 :                                     pagesize_2pow)), (npages << pagesize_2pow),
        3648                 :                                     MADV_FREE);
        3649                 : #  ifdef MALLOC_DOUBLE_PURGE
        3650                 :                                 madvised = true;
        3651                 : #  endif
        3652                 : #endif
        3653                 : #ifdef MALLOC_STATS
        3654            1770 :                                 arena->stats.nmadvise++;
        3655            1770 :                                 arena->stats.purged += npages;
        3656                 : #endif
        3657            1770 :                                 if (arena->ndirty <= (opt_dirty_max >> 1))
        3658             165 :                                         break;
        3659                 :                         }
        3660                 :                 }
        3661                 :
        3662             914 :                 if (chunk->ndirty == 0) {
        3663             860 :                         arena_chunk_tree_dirty_remove(&arena->chunks_dirty,
        3664                 :                             chunk);
        3665                 :                 }
        3666                 : #ifdef MALLOC_DOUBLE_PURGE
        3667                 :                 if (madvised) {
        3668                 :                         /* The chunk might already be in the list, but this
        3669                 :                          * makes sure it's at the front. */
        3670                 :                         LinkedList_Remove(&chunk->chunks_madvised_elem);
        3671                 :                         LinkedList_InsertHead(&arena->chunks_madvised, &chunk->chunks_madvised_elem);
        3672                 :                 }
        3673                 : #endif
        3674                 :         }
        3675             165 : }
        3676                 :
        3677                 : static void
        3678         3372371 : arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
        3679                 : {
        3680                 :         arena_chunk_t *chunk;
        3681                 :         size_t size, run_ind, run_pages;
        3682                 :
        3683         3372371 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
        3684         3372371 :         run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk)
        3685                 :             >> pagesize_2pow);
        3686         3372371 :         assert(run_ind >= arena_chunk_header_npages);
        3687         3372371 :         assert(run_ind < chunk_npages);
        3688         3372371 :         if ((chunk->map[run_ind].bits & CHUNK_MAP_LARGE) != 0)
        3689          637739 :                 size = chunk->map[run_ind].bits & ~pagesize_mask;
        3690                 :         else
        3691         2734632 :                 size = run->bin->run_size;
        3692         3372371 :         run_pages = (size >> pagesize_2pow);
        3693                 :
        3694                 :         /* Mark pages as unallocated in the chunk map. */
        3695         3372371 :         if (dirty) {
        3696                 :                 size_t i;
        3697                 :
        3698        20482931 :                 for (i = 0; i < run_pages; i++) {
        3699        17110560 :                         assert((chunk->map[run_ind + i].bits & CHUNK_MAP_DIRTY)
        3700                 :                             == 0);
        3701        17110560 :                         chunk->map[run_ind + i].bits = CHUNK_MAP_DIRTY;
        3702                 :                 }
        3703                 :
        3704         3372371 :                 if (chunk->ndirty == 0) {
        3705          157649 :                         arena_chunk_tree_dirty_insert(&arena->chunks_dirty,
        3706                 :                             chunk);
        3707                 :                 }
        3708         3372371 :                 chunk->ndirty += run_pages;
        3709         3372371 :                 arena->ndirty += run_pages;
        3710                 :         } else {
        3711                 :                 size_t i;
        3712                 :
        3713               0 :                 for (i = 0; i < run_pages; i++) {
        3714               0 :                         chunk->map[run_ind + i].bits &= ~(CHUNK_MAP_LARGE |
        3715                 :                             CHUNK_MAP_ALLOCATED);
        3716                 :                 }
        3717                 :         }
        3718         3372371 :         chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
        3719                 :             pagesize_mask);
        3720         6744742 :         chunk->map[run_ind+run_pages-1].bits = size |
        3721         3372371 :             (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
        3722                 :
        3723                 :         /* Try to coalesce forward. */
        3724         6731503 :         if (run_ind + run_pages < chunk_npages &&
        3725         3359132 :             (chunk->map[run_ind+run_pages].bits & CHUNK_MAP_ALLOCATED) == 0) {
        3726         2801107 :                 size_t nrun_size = chunk->map[run_ind+run_pages].bits &
        3727                 :                     ~pagesize_mask;
        3728                 :
        3729                 :                 /*
        3730                 :                  * Remove successor from runs_avail; the coalesced run is
        3731                 :                  * inserted later.
        3732                 :                  */
        3733         2801107 :                 arena_avail_tree_remove(&arena->runs_avail,
        3734         2801107 :                     &chunk->map[run_ind+run_pages]);
        3735                 :
        3736         2801107 :                 size += nrun_size;
        3737         2801107 :                 run_pages = size >> pagesize_2pow;
        3738                 :
        3739         2801107 :                 assert((chunk->map[run_ind+run_pages-1].bits & ~pagesize_mask)
        3740                 :                     == nrun_size);
        3741         2801107 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
        3742                 :                     pagesize_mask);
        3743         5602214 :                 chunk->map[run_ind+run_pages-1].bits = size |
        3744         2801107 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
        3745                 :         }
        3746                 :
        3747                 :         /* Try to coalesce backward. */
        3748         3372371 :         if (run_ind > arena_chunk_header_npages && (chunk->map[run_ind-1].bits &
        3749                 :             CHUNK_MAP_ALLOCATED) == 0) {
        3750          446690 :                 size_t prun_size = chunk->map[run_ind-1].bits & ~pagesize_mask;
        3751                 :
        3752          446690 :                 run_ind -= prun_size >> pagesize_2pow;
        3753                 :
        3754                 :                 /*
        3755                 :                  * Remove predecessor from runs_avail; the coalesced run is
        3756                 :                  * inserted later.
        3757                 :                  */
        3758          446690 :                 arena_avail_tree_remove(&arena->runs_avail,
        3759          446690 :                     &chunk->map[run_ind]);
        3760                 :
        3761          446690 :                 size += prun_size;
        3762          446690 :                 run_pages = size >> pagesize_2pow;
        3763                 :
        3764          446690 :                 assert((chunk->map[run_ind].bits & ~pagesize_mask) ==
        3765                 :                     prun_size);
        3766          446690 :                 chunk->map[run_ind].bits = size | (chunk->map[run_ind].bits &
        3767                 :                     pagesize_mask);
        3768          893380 :                 chunk->map[run_ind+run_pages-1].bits = size |
        3769          446690 :                     (chunk->map[run_ind+run_pages-1].bits & pagesize_mask);
        3770                 :         }
        3771                 :
        3772                 :         /* Insert into runs_avail, now that coalescing is complete. */
        3773         3372371 :         arena_avail_tree_insert(&arena->runs_avail, &chunk->map[run_ind]);
        3774                 :
        3775                 :         /* Deallocate chunk if it is now completely unused. */
        3776         6744742 :         if ((chunk->map[arena_chunk_header_npages].bits & (~pagesize_mask |
        3777         3372371 :             CHUNK_MAP_ALLOCATED)) == arena_maxclass)
        3778           30235 :                 arena_chunk_dealloc(arena, chunk);
        3779                 :
        3780                 :         /* Enforce opt_dirty_max. */
        3781         3372371 :         if (arena->ndirty > opt_dirty_max)
        3782             165 :                 arena_purge(arena);
        3783         3372371 : }
        3784                 :
        3785                 : static void
        3786               0 : arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
        3787                 :     size_t oldsize, size_t newsize)
        3788                 : {
        3789               0 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
        3790               0 :         size_t head_npages = (oldsize - newsize) >> pagesize_2pow;
        3791                 :
        3792               0 :         assert(oldsize > newsize);
        3793                 :
        3794                 :         /*
        3795                 :          * Update the chunk map so that arena_run_dalloc() can treat the
        3796                 :          * leading run as separately allocated.
        3797                 :          */
        3798               0 :         chunk->map[pageind].bits = (oldsize - newsize) | CHUNK_MAP_LARGE |
        3799                 :             CHUNK_MAP_ALLOCATED;
        3800               0 :         chunk->map[pageind+head_npages].bits = newsize | CHUNK_MAP_LARGE |
        3801                 :             CHUNK_MAP_ALLOCATED;
        3802                 :
        3803               0 :         arena_run_dalloc(arena, run, false);
        3804               0 : }
        3805                 :
        3806                 : static void
        3807             117 : arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
        3808                 :     size_t oldsize, size_t newsize, bool dirty)
        3809                 : {
        3810             117 :         size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> pagesize_2pow;
        3811             117 :         size_t npages = newsize >> pagesize_2pow;
        3812                 :
        3813             117 :         assert(oldsize > newsize);
        3814                 :
        3815                 :         /*
        3816                 :          * Update the chunk map so that arena_run_dalloc() can treat the
        3817                 :          * trailing run as separately allocated.
        3818                 :          */
        3819             117 :         chunk->map[pageind].bits = newsize | CHUNK_MAP_LARGE |
        3820                 :             CHUNK_MAP_ALLOCATED;
        3821             234 :         chunk->map[pageind+npages].bits = (oldsize - newsize) | CHUNK_MAP_LARGE
        3822             117 :             | CHUNK_MAP_ALLOCATED;
        3823                 :
        3824             117 :         arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
        3825                 :             dirty);
        3826             117 : }
        3827                 :
        3828                 : static arena_run_t *
        3829         3182856 : arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
        3830                 : {
        3831                 :         arena_chunk_map_t *mapelm;
        3832                 :         arena_run_t *run;
        3833                 :         unsigned i, remainder;
        3834                 :
        3835                 :         /* Look for a usable run. */
        3836         3182856 :         mapelm = arena_run_tree_first(&bin->runs);
        3837         3182856 :         if (mapelm != NULL) {
        3838                 :                 /* run is guaranteed to have available space. */
        3839          130875 :                 arena_run_tree_remove(&bin->runs, mapelm);
        3840          130875 :                 run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
        3841                 : #ifdef MALLOC_STATS
        3842          130875 :                 bin->stats.reruns++;
        3843                 : #endif
        3844          130875 :                 return (run);
        3845                 :         }
        3846                 :         /* No existing runs have any space available. */
        3847                 :
        3848                 :         /* Allocate a new run. */
        3849         3051981 :         run = arena_run_alloc(arena, bin, bin->run_size, false, false);
        3850         3051981 :         if (run == NULL)
        3851               0 :                 return (NULL);
        3852                 :         /*
        3853                 :          * Don't initialize if a race in arena_run_alloc() allowed an existing
        3854                 :          * run to become usable.
        3855                 :          */
        3856         3051981 :         if (run == bin->runcur)
        3857               0 :                 return (run);
        3858                 :
        3859                 :         VALGRIND_MALLOCLIKE_BLOCK(run, sizeof(arena_run_t) + (sizeof(unsigned) *
        3860                 :             (bin->regs_mask_nelms - 1)), 0, false);
        3861                 :
        3862                 :         /* Initialize run internals. */
        3863         3051981 :         run->bin = bin;
        3864                 :
        3865         7742566 :         for (i = 0; i < bin->regs_mask_nelms - 1; i++)
        3866         4690585 :                 run->regs_mask[i] = UINT_MAX;
        3867         3051981 :         remainder = bin->nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1);
        3868         3051981 :         if (remainder == 0)
        3869               0 :                 run->regs_mask[i] = UINT_MAX;
        3870                 :         else {
        3871                 :                 /* The last element has spare bits that need to be unset. */
        3872         3051981 :                 run->regs_mask[i] = (UINT_MAX >> ((1U << (SIZEOF_INT_2POW + 3))
        3873         3051981 :                     - remainder));
        3874                 :         }
        3875                 :
        3876         3051981 :         run->regs_minelm = 0;
        3877                 :
        3878         3051981 :         run->nfree = bin->nregs;
        3879                 : #ifdef MALLOC_DEBUG
        3880         3051981 :         run->magic = ARENA_RUN_MAGIC;
        3881                 : #endif
        3882                 :
        3883                 : #ifdef MALLOC_STATS
        3884         3051981 :         bin->stats.nruns++;
        3885         3051981 :         bin->stats.curruns++;
        3886         3051981 :         if (bin->stats.curruns > bin->stats.highruns)
        3887          678835 :                 bin->stats.highruns = bin->stats.curruns;
        3888                 : #endif
        3889         3051981 :         return (run);
        3890                 : }
        3891                 :
        3892                 : /* bin->runcur must have space available before this function is called. */
        3893                 : static inline void *
        3894        45735071 : arena_bin_malloc_easy(arena_t *arena, arena_bin_t *bin, arena_run_t *run)
        3895                 : {
        3896                 :         void *ret;
        3897                 :
        3898        45735071 :         assert(run->magic == ARENA_RUN_MAGIC);
        3899        45735071 :         assert(run->nfree > 0);
        3900                 :
        3901        45735071 :         ret = arena_run_reg_alloc(run, bin);
        3902        45735071 :         assert(ret != NULL);
        3903        45735071 :         run->nfree--;
        3904                 :
        3905        45735071 :         return (ret);
        3906                 : }
        3907                 :
        3908                 : /* Re-fill bin->runcur, then call arena_bin_malloc_easy(). */
        3909                 : static void *
        3910         3182856 : arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
        3911                 : {
        3912                 :
        3913         3182856 :         bin->runcur = arena_bin_nonfull_run_get(arena, bin);
        3914         3182856 :         if (bin->runcur == NULL)
        3915               0 :                 return (NULL);
        3916         3182856 :         assert(bin->runcur->magic == ARENA_RUN_MAGIC);
        3917         3182856 :         assert(bin->runcur->nfree > 0);
        3918                 :
        3919         3182856 :         return (arena_bin_malloc_easy(arena, bin, bin->runcur));
        3920                 : }
        3921                 :
        3922                 : /*
        3923                 :  * Calculate bin->run_size such that it meets the following constraints:
        3924                 :  *
        3925                 :  *   *) bin->run_size >= min_run_size
        3926                 :  *   *) bin->run_size <= arena_maxclass
        3927                 :  *   *) bin->run_size <= RUN_MAX_SMALL
        3928                 :  *   *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed).
        3929                 :  *
        3930                 :  * bin->nregs, bin->regs_mask_nelms, and bin->reg0_offset are
        3931                 :  * also calculated here, since these settings are all interdependent.
        3932                 :  */
        3933                 : static size_t
        3934          672084 : arena_bin_run_size_calc(arena_bin_t *bin, size_t min_run_size)
        3935                 : {
        3936                 :         size_t try_run_size, good_run_size;
        3937                 :         unsigned good_nregs, good_mask_nelms, good_reg0_offset;
        3938                 :         unsigned try_nregs, try_mask_nelms, try_reg0_offset;
        3939                 :
        3940          672084 :         assert(min_run_size >= pagesize);
        3941          672084 :         assert(min_run_size <= arena_maxclass);
        3942          672084 :         assert(min_run_size <= RUN_MAX_SMALL);
        3943                 :
        3944                 :         /*
        3945                 :          * Calculate known-valid settings before entering the run_size
        3946                 :          * expansion loop, so that the first part of the loop always copies
        3947                 :          * valid settings.
        3948                 :          *
        3949                 :          * The do..while loop iteratively reduces the number of regions until
        3950                 :          * the run header and the regions no longer overlap.  A closed formula
        3951                 :          * would be quite messy, since there is an interdependency between the
        3952                 :          * header's mask length and the number of regions.
        3953                 :          */
        3954          672084 :         try_run_size = min_run_size;
        3955          672084 :         try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin->reg_size)
        3956                 :             + 1; /* Counter-act try_nregs-- in loop. */
        3957                 :         do {
        3958         1381506 :                 try_nregs--;
        3959         2763012 :                 try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
        3960         1381506 :                     ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ? 1 : 0);
        3961         1381506 :                 try_reg0_offset = try_run_size - (try_nregs * bin->reg_size);
        3962         1381506 :         } while (sizeof(arena_run_t) + (sizeof(unsigned) * (try_mask_nelms - 1))
        3963         1381506 :             > try_reg0_offset);
        3964                 :
        3965                 :         /* run_size expansion loop. */
        3966                 :         do {
        3967                 :                 /*
        3968                 :                  * Copy valid settings before trying more aggressive settings.
        3969                 :                  */
        3970          802767 :                 good_run_size = try_run_size;
        3971          802767 :                 good_nregs = try_nregs;
        3972          802767 :                 good_mask_nelms = try_mask_nelms;
        3973          802767 :                 good_reg0_offset = try_reg0_offset;
        3974                 :
        3975                 :                 /* Try more aggressive settings. */
        3976          802767 :                 try_run_size += pagesize;
        3977         1605534 :                 try_nregs = ((try_run_size - sizeof(arena_run_t)) /
        3978          802767 :                     bin->reg_size) + 1; /* Counter-act try_nregs-- in loop. */
        3979                 :                 do {
        3980         2314956 :                         try_nregs--;
        3981         4629912 :                         try_mask_nelms = (try_nregs >> (SIZEOF_INT_2POW + 3)) +
        3982         2314956 :                             ((try_nregs & ((1U << (SIZEOF_INT_2POW + 3)) - 1)) ?
        3983                 :                             1 : 0);
        3984         2314956 :                         try_reg0_offset = try_run_size - (try_nregs *
        3985         2314956 :                             bin->reg_size);
        3986         2314956 :                 } while (sizeof(arena_run_t) + (sizeof(unsigned) *
        3987         2314956 :                     (try_mask_nelms - 1)) > try_reg0_offset);
        3988         1605534 :         } while (try_run_size <= arena_maxclass && try_run_size <= RUN_MAX_SMALL
        3989          746760 :             && RUN_MAX_OVRHD * (bin->reg_size << 3) > RUN_MAX_OVRHD_RELAX
        3990         1512189 :             && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size);
        3991                 :
        3992          672084 :         assert(sizeof(arena_run_t) + (sizeof(unsigned) * (good_mask_nelms - 1))
        3993                 :             <= good_reg0_offset);
        3994          672084 :         assert((good_mask_nelms << (SIZEOF_INT_2POW + 3)) >= good_nregs);
        3995                 :
        3996                 :         /* Copy final settings. */
        3997          672084 :         bin->run_size = good_run_size;
        3998          672084 :         bin->nregs = good_nregs;
        3999          672084 :         bin->regs_mask_nelms = good_mask_nelms;
        4000          672084 :         bin->reg0_offset = good_reg0_offset;
        4001                 :
        4002          672084 :         return (good_run_size);
        4003                 : }
        4004                 :
        4005                 : #ifdef MALLOC_BALANCE
        4006                 : static inline void
        4007                 : arena_lock_balance(arena_t *arena)
        4008                 : {
        4009                 :         unsigned contention;
        4010                 :
        4011                 :         contention = malloc_spin_lock(&arena->lock);
        4012                 :         if (narenas > 1) {
        4013                 :                 /*
        4014                 :                  * Calculate the exponentially averaged contention for this
        4015                 :                  * arena.  Due to integer math always rounding down, this value
        4016                 :                  * decays somewhat faster then normal.
        4017                 :                  */
        4018                 :                 arena->contention = (((uint64_t)arena->contention
        4019                 :                     * (uint64_t)((1U << BALANCE_ALPHA_INV_2POW)-1))
        4020                 :                     + (uint64_t)contention) >> BALANCE_ALPHA_INV_2POW;
        4021                 :                 if (arena->contention >= opt_balance_threshold)
        4022                 :                         arena_lock_balance_hard(arena);
        4023                 :         }
        4024                 : }
        4025                 :
        4026                 : static void
        4027                 : arena_lock_balance_hard(arena_t *arena)
        4028                 : {
        4029                 :         uint32_t ind;
        4030                 :
        4031                 :         arena->contention = 0;
        4032                 : #ifdef MALLOC_STATS
        4033                 :         arena->stats.nbalance++;
        4034                 : #endif
        4035                 :         ind = PRN(balance, narenas_2pow);
        4036                 :         if (arenas[ind] != NULL) {
        4037                 : #ifdef MOZ_MEMORY_WINDOWS
        4038                 :                 TlsSetValue(tlsIndex, arenas[ind]);
        4039                 : #else
        4040                 :                 arenas_map = arenas[ind];
        4041                 : #endif
        4042                 :         } else {
        4043                 :                 malloc_spin_lock(&arenas_lock);
        4044                 :                 if (arenas[ind] != NULL) {
        4045                 : #ifdef MOZ_MEMORY_WINDOWS
        4046                 :                         TlsSetValue(tlsIndex, arenas[ind]);
        4047                 : #else
        4048                 :                         arenas_map = arenas[ind];
        4049                 : #endif
        4050                 :                 } else {
        4051                 : #ifdef MOZ_MEMORY_WINDOWS
        4052                 :                         TlsSetValue(tlsIndex, arenas_extend(ind));
        4053                 : #else
        4054                 :                         arenas_map = arenas_extend(ind);
        4055                 : #endif
        4056                 :                 }
        4057                 :                 malloc_spin_unlock(&arenas_lock);
        4058                 :         }
        4059                 : }
        4060                 : #endif
        4061                 :
        4062                 : static inline void *
        4063        45735071 : arena_malloc_small(arena_t *arena, size_t size, bool zero)
        4064                 : {
        4065                 :         void *ret;
        4066                 :         arena_bin_t *bin;
        4067                 :         arena_run_t *run;
        4068                 :
        4069        45735071 :         if (size < small_min) {
        4070                 :                 /* Tiny. */
        4071        11588699 :                 size = pow2_ceil(size);
        4072        11588699 :                 bin = &arena->bins[ffs((int)(size >> (TINY_MIN_2POW +
        4073                 :                     1)))];
        4074                 : #if (!defined(NDEBUG) || defined(MALLOC_STATS))
        4075                 :                 /*
        4076                 :                  * Bin calculation is always correct, but we may need
        4077                 :                  * to fix size for the purposes of assertions and/or
        4078                 :                  * stats accuracy.
        4079                 :                  */
        4080        11588699 :                 if (size < (1U << TINY_MIN_2POW))
        4081           39317 :                         size = (1U << TINY_MIN_2POW);
        4082                 : #endif
        4083        34146372 :         } else if (size <= small_max) {
        4084                 :                 /* Quantum-spaced. */
        4085        32199741 :                 size = QUANTUM_CEILING(size);
        4086        32199741 :                 bin = &arena->bins[ntbins + (size >> opt_quantum_2pow)
        4087                 :                     - 1];
        4088                 :         } else {
        4089                 :                 /* Sub-page. */
        4090         1946631 :                 size = pow2_ceil(size);
        4091         3893262 :                 bin = &arena->bins[ntbins + nqbins
        4092         1946631 :                     + (ffs((int)(size >> opt_small_max_2pow)) - 2)];
        4093                 :         }
        4094        45735071 :         assert(size == bin->reg_size);
        4095                 :
        4096                 : #ifdef MALLOC_BALANCE
        4097                 :         arena_lock_balance(arena);
        4098                 : #else
        4099        45735071 :         malloc_spin_lock(&arena->lock);
        4100                 : #endif
        4101        45735071 :         if ((run = bin->runcur) != NULL && run->nfree > 0)
        4102        42552215 :                 ret = arena_bin_malloc_easy(arena, bin, run);
        4103                 :         else
        4104         3182856 :                 ret = arena_bin_malloc_hard(arena, bin);
        4105                 :
        4106        45735071 :         if (ret == NULL) {
        4107               0 :                 malloc_spin_unlock(&arena->lock);
        4108               0 :                 return (NULL);
        4109                 :         }
        4110                 :
        4111                 : #ifdef MALLOC_STATS
        4112        45735071 :         bin->stats.nrequests++;
        4113        45735071 :         arena->stats.nmalloc_small++;
        4114        45735071 :         arena->stats.allocated_small += size;
        4115                 : #endif
        4116        45735071 :         malloc_spin_unlock(&arena->lock);
        4117                 :
        4118                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
        4119        45735071 :         if (zero == false) {
        4120                 : #ifdef MALLOC_FILL
        4121        43672870 :                 if (opt_junk)
        4122        43672870 :                         memset(ret, 0xa5, size);
        4123               0 :                 else if (opt_zero)
        4124               0 :                         memset(ret, 0, size);
        4125                 : #endif
        4126                 :         } else
        4127         2062201 :                 memset(ret, 0, size);
        4128                 :
        4129        45735071 :         return (ret);
        4130                 : }
        4131                 :
        4132                 : static void *
        4133          637631 : arena_malloc_large(arena_t *arena, size_t size, bool zero)
        4134                 : {
        4135                 :         void *ret;
        4136                 :
        4137                 :         /* Large allocation. */
        4138          637631 :         size = PAGE_CEILING(size);
        4139                 : #ifdef MALLOC_BALANCE
        4140                 :         arena_lock_balance(arena);
        4141                 : #else
        4142          637631 :         malloc_spin_lock(&arena->lock);
        4143                 : #endif
        4144          637631 :         ret = (void *)arena_run_alloc(arena, NULL, size, true, zero);
        4145          637631 :         if (ret == NULL) {
        4146               0 :                 malloc_spin_unlock(&arena->lock);
        4147               0 :                 return (NULL);
        4148                 :         }
        4149                 : #ifdef MALLOC_STATS
        4150          637631 :         arena->stats.nmalloc_large++;
        4151          637631 :         arena->stats.allocated_large += size;
        4152                 : #endif
        4153          637631 :         malloc_spin_unlock(&arena->lock);
        4154                 :
        4155                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, zero);
        4156          637631 :         if (zero == false) {
        4157                 : #ifdef MALLOC_FILL
        4158          443449 :                 if (opt_junk)
        4159          443449 :                         memset(ret, 0xa5, size);
        4160               0 :                 else if (opt_zero)
        4161               0 :                         memset(ret, 0, size);
        4162                 : #endif
        4163                 :         }
        4164                 :
        4165          637631 :         return (ret);
        4166                 : }
        4167                 :
        4168                 : static inline void *
        4169        46372702 : arena_malloc(arena_t *arena, size_t size, bool zero)
        4170                 : {
        4171                 :
        4172        46372702 :         assert(arena != NULL);
        4173        46372702 :         assert(arena->magic == ARENA_MAGIC);
        4174        46372702 :         assert(size != 0);
        4175        46372702 :         assert(QUANTUM_CEILING(size) <= arena_maxclass);
        4176                 :
        4177        46372702 :         if (size <= bin_maxclass) {
        4178        45735071 :                 return (arena_malloc_small(arena, size, zero));
        4179                 :         } else
        4180          637631 :                 return (arena_malloc_large(arena, size, zero));
        4181                 : }
        4182                 :
        4183                 : static inline void *
        4184        41858280 : imalloc(size_t size)
        4185                 : {
        4186                 :
        4187        41858280 :         assert(size != 0);
        4188                 :
        4189        41858280 :         if (size <= arena_maxclass)
        4190        41855450 :                 return (arena_malloc(choose_arena(), size, false));
        4191                 :         else
        4192            2830 :                 return (huge_malloc(size, false));
        4193                 : }
        4194                 :
        4195                 : static inline void *
        4196         2256419 : icalloc(size_t size)
        4197                 : {
        4198                 :
        4199         2256419 :         if (size <= arena_maxclass)
        4200         2256383 :                 return (arena_malloc(choose_arena(), size, true));
        4201                 :         else
        4202              36 :                 return (huge_malloc(size, true));
        4203                 : }
        4204                 :
        4205                 : /* Only handles large allocations that require more than page alignment. */
        4206                 : static void *
        4207               0 : arena_palloc(arena_t *arena, size_t alignment, size_t size, size_t alloc_size)
        4208                 : {
        4209                 :         void *ret;
        4210                 :         size_t offset;
        4211                 :         arena_chunk_t *chunk;
        4212                 :
        4213               0 :         assert((size & pagesize_mask) == 0);
        4214               0 :         assert((alignment & pagesize_mask) == 0);
        4215                 :
        4216                 : #ifdef MALLOC_BALANCE
        4217                 :         arena_lock_balance(arena);
        4218                 : #else
        4219               0 :         malloc_spin_lock(&arena->lock);
        4220                 : #endif
        4221               0 :         ret = (void *)arena_run_alloc(arena, NULL, alloc_size, true, false);
        4222               0 :         if (ret == NULL) {
        4223               0 :                 malloc_spin_unlock(&arena->lock);
        4224               0 :                 return (NULL);
        4225                 :         }
        4226                 :
        4227               0 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
        4228                 :
        4229               0 :         offset = (uintptr_t)ret & (alignment - 1);
        4230               0 :         assert((offset & pagesize_mask) == 0);
        4231               0 :         assert(offset < alloc_size);
        4232               0 :         if (offset == 0)
        4233               0 :                 arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, alloc_size, size, false);
        4234                 :         else {
        4235                 :                 size_t leadsize, trailsize;
        4236                 :
        4237               0 :                 leadsize = alignment - offset;
        4238               0 :                 if (leadsize > 0) {
        4239               0 :                         arena_run_trim_head(arena, chunk, (arena_run_t*)ret, alloc_size,
        4240                 :                             alloc_size - leadsize);
        4241               0 :                         ret = (void *)((uintptr_t)ret + leadsize);
        4242                 :                 }
        4243                 :
        4244               0 :                 trailsize = alloc_size - leadsize - size;
        4245               0 :                 if (trailsize != 0) {
        4246                 :                         /* Trim trailing space. */
        4247               0 :                         assert(trailsize < alloc_size);
        4248               0 :                         arena_run_trim_tail(arena, chunk, (arena_run_t*)ret, size + trailsize,
        4249                 :                             size, false);
        4250                 :                 }
        4251                 :         }
        4252                 :
        4253                 : #ifdef MALLOC_STATS
        4254               0 :         arena->stats.nmalloc_large++;
        4255               0 :         arena->stats.allocated_large += size;
        4256                 : #endif
        4257               0 :         malloc_spin_unlock(&arena->lock);
        4258                 :
        4259                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, size, 0, false);
        4260                 : #ifdef MALLOC_FILL
        4261               0 :         if (opt_junk)
        4262               0 :                 memset(ret, 0xa5, size);
        4263               0 :         else if (opt_zero)
        4264               0 :                 memset(ret, 0, size);
        4265                 : #endif
        4266               0 :         return (ret);
        4267                 : }
        4268                 :
        4269                 : static inline void *
        4270               0 : ipalloc(size_t alignment, size_t size)
        4271                 : {
        4272                 :         void *ret;
        4273                 :         size_t ceil_size;
        4274                 :
        4275                 :         /*
        4276                 :          * Round size up to the nearest multiple of alignment.
        4277                 :          *
        4278                 :          * This done, we can take advantage of the fact that for each small
        4279                 :          * size class, every object is aligned at the smallest power of two
        4280                 :          * that is non-zero in the base two representation of the size.  For
        4281                 :          * example:
        4282                 :          *
        4283                 :          *   Size |   Base 2 | Minimum alignment
        4284                 :          *   -----+----------+------------------
        4285                 :          *     96 |  1100000 |  32
        4286                 :          *    144 | 10100000 |  32
        4287                 :          *    192 | 11000000 |  64
        4288                 :          *
        4289                 :          * Depending on runtime settings, it is possible that arena_malloc()
        4290                 :          * will further round up to a power of two, but that never causes
        4291                 :          * correctness issues.
        4292                 :          */
        4293               0 :         ceil_size = (size + (alignment - 1)) & (-alignment);
        4294                 :         /*
        4295                 :          * (ceil_size < size) protects against the combination of maximal
        4296                 :          * alignment and size greater than maximal alignment.
        4297                 :          */
        4298               0 :         if (ceil_size < size) {
        4299                 :                 /* size_t overflow. */
        4300               0 :                 return (NULL);
        4301                 :         }
        4302                 :
        4303               0 :         if (ceil_size <= pagesize || (alignment <= pagesize
        4304               0 :             && ceil_size <= arena_maxclass))
        4305               0 :                 ret = arena_malloc(choose_arena(), ceil_size, false);
        4306                 :         else {
        4307                 :                 size_t run_size;
        4308                 :
        4309                 :                 /*
        4310                 :                  * We can't achieve sub-page alignment, so round up alignment
        4311                 :                  * permanently; it makes later calculations simpler.
        4312                 :                  */
        4313               0 :                 alignment = PAGE_CEILING(alignment);
        4314               0 :                 ceil_size = PAGE_CEILING(size);
        4315                 :                 /*
        4316                 :                  * (ceil_size < size) protects against very large sizes within
        4317                 :                  * pagesize of SIZE_T_MAX.
        4318                 :                  *
        4319                 :                  * (ceil_size + alignment < ceil_size) protects against the
        4320                 :                  * combination of maximal alignment and ceil_size large enough
        4321                 :                  * to cause overflow.  This is similar to the first overflow
        4322                 :                  * check above, but it needs to be repeated due to the new
        4323                 :                  * ceil_size value, which may now be *equal* to maximal
        4324                 :                  * alignment, whereas before we only detected overflow if the
        4325                 :                  * original size was *greater* than maximal alignment.
        4326                 :                  */
        4327               0 :                 if (ceil_size < size || ceil_size + alignment < ceil_size) {
        4328                 :                         /* size_t overflow. */
        4329               0 :                         return (NULL);
        4330                 :                 }
        4331                 :
        4332                 :                 /*
        4333                 :                  * Calculate the size of the over-size run that arena_palloc()
        4334                 :                  * would need to allocate in order to guarantee the alignment.
        4335                 :                  */
        4336               0 :                 if (ceil_size >= alignment)
        4337               0 :                         run_size = ceil_size + alignment - pagesize;
        4338                 :                 else {
        4339                 :                         /*
        4340                 :                          * It is possible that (alignment << 1) will cause
        4341                 :                          * overflow, but it doesn't matter because we also
        4342                 :                          * subtract pagesize, which in the case of overflow
        4343                 :                          * leaves us with a very large run_size.  That causes
        4344                 :                          * the first conditional below to fail, which means
        4345                 :                          * that the bogus run_size value never gets used for
        4346                 :                          * anything important.
        4347                 :                          */
        4348               0 :                         run_size = (alignment << 1) - pagesize;
        4349                 :                 }
        4350                 :
        4351               0 :                 if (run_size <= arena_maxclass) {
        4352               0 :                         ret = arena_palloc(choose_arena(), alignment, ceil_size,
        4353                 :                             run_size);
        4354               0 :                 } else if (alignment <= chunksize)
        4355               0 :                         ret = huge_malloc(ceil_size, false);
        4356                 :                 else
        4357               0 :                         ret = huge_palloc(alignment, ceil_size);
        4358                 :         }
        4359                 :
        4360               0 :         assert(((uintptr_t)ret & (alignment - 1)) == 0);
        4361               0 :         return (ret);
        4362                 : }
        4363                 :
        4364                 : /* Return the size of the allocation pointed to by ptr. */
        4365                 : static size_t
        4366         2300043 : arena_salloc(const void *ptr)
        4367                 : {
        4368                 :         size_t ret;
        4369                 :         arena_chunk_t *chunk;
        4370                 :         size_t pageind, mapbits;
        4371                 :
        4372         2300043 :         assert(ptr != NULL);
        4373         2300043 :         assert(CHUNK_ADDR2BASE(ptr) != ptr);
        4374                 :
        4375         2300043 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
        4376         2300043 :         pageind = (((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow);
        4377         2300043 :         mapbits = chunk->map[pageind].bits;
        4378         2300043 :         assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
        4379         2300043 :         if ((mapbits & CHUNK_MAP_LARGE) == 0) {
        4380         2271520 :                 arena_run_t *run = (arena_run_t *)(mapbits & ~pagesize_mask);
        4381         2271520 :                 assert(run->magic == ARENA_RUN_MAGIC);
        4382         2271520 :                 ret = run->bin->reg_size;
        4383                 :         } else {
        4384           28523 :                 ret = mapbits & ~pagesize_mask;
        4385           28523 :                 assert(ret != 0);
        4386                 :         }
        4387                 :
        4388         2300043 :         return (ret);
        4389                 : }
        4390                 :
        4391                 : #if (defined(MALLOC_VALIDATE) || defined(MOZ_MEMORY_DARWIN))
        4392                 : /*
        4393                 :  * Validate ptr before assuming that it points to an allocation.  Currently,
        4394                 :  * the following validation is performed:
        4395                 :  *
        4396                 :  * + Check that ptr is not NULL.
        4397                 :  *
        4398                 :  * + Check that ptr lies within a mapped chunk.
        4399                 :  */
        4400                 : static inline size_t
        4401               0 : isalloc_validate(const void *ptr)
        4402                 : {
        4403                 :         arena_chunk_t *chunk;
        4404                 :
        4405               0 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
        4406               0 :         if (chunk == NULL)
        4407               0 :                 return (0);
        4408                 :
        4409               0 :         if (malloc_rtree_get(chunk_rtree, (uintptr_t)chunk) == NULL)
        4410               0 :                 return (0);
        4411                 :
        4412               0 :         if (chunk != ptr) {
        4413               0 :                 assert(chunk->arena->magic == ARENA_MAGIC);
        4414               0 :                 return (arena_salloc(ptr));
        4415                 :         } else {
        4416                 :                 size_t ret;
        4417                 :                 extent_node_t *node;
        4418                 :                 extent_node_t key;
        4419                 :
        4420                 :                 /* Chunk. */
        4421               0 :                 key.addr = (void *)chunk;
        4422               0 :                 malloc_mutex_lock(&huge_mtx);
        4423               0 :                 node = extent_tree_ad_search(&huge, &key);
        4424               0 :                 if (node != NULL)
        4425               0 :                         ret = node->size;
        4426                 :                 else
        4427               0 :                         ret = 0;
        4428               0 :                 malloc_mutex_unlock(&huge_mtx);
        4429               0 :                 return (ret);
        4430                 :         }
        4431                 : }
        4432                 : #endif
        4433                 :
        4434                 : static inline size_t
        4435         2301663 : isalloc(const void *ptr)
        4436                 : {
        4437                 :         size_t ret;
        4438                 :         arena_chunk_t *chunk;
        4439                 :
        4440         2301663 :         assert(ptr != NULL);
        4441                 :
        4442         2301663 :         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
        4443         2301663 :         if (chunk != ptr) {
        4444                 :                 /* Region. */
        4445         2300043 :                 assert(chunk->arena->magic == ARENA_MAGIC);
        4446                 :
        4447         2300043 :                 ret = arena_salloc(ptr);
        4448                 :         } else {
        4449                 :                 extent_node_t *node, key;
        4450                 :
        4451                 :                 /* Chunk (huge allocation). */
        4452                 :
        4453            1620 :                 malloc_mutex_lock(&huge_mtx);
        4454                 :
        4455                 :                 /* Extract from tree of huge allocations. */
        4456            1620 :                 key.addr = __DECONST(void *, ptr);
        4457            1620 :                 node = extent_tree_ad_search(&huge, &key);
        4458            1620 :                 assert(node != NULL);
        4459                 :
        4460            1620 :                 ret = node->size;
        4461                 :
        4462            1620 :                 malloc_mutex_unlock(&huge_mtx);
        4463                 :         }
        4464                 :
        4465         2301663 :         return (ret);
        4466                 : }
        4467                 :
        4468                 : static inline void
        4469        43659702 : arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
        4470                 :     arena_chunk_map_t *mapelm)
        4471                 : {
        4472                 :         arena_run_t *run;
        4473                 :         arena_bin_t *bin;
        4474                 :         size_t size;
        4475                 :
        4476        43659702 :         run = (arena_run_t *)(mapelm->bits & ~pagesize_mask);
        4477        43659702 :         assert(run->magic == ARENA_RUN_MAGIC);
        4478        43659702 :         bin = run->bin;
        4479        43659702 :         size = bin->reg_size;
        4480                 :
        4481                 : #ifdef MALLOC_FILL
        4482        43659702 :         if (opt_junk)
        4483        43659702 :                 memset(ptr, 0x5a, size);
        4484                 : #endif
        4485                 :
        4486        43659702 :         arena_run_reg_dalloc(run, bin, ptr, size);
        4487        43659702 :         run->nfree++;
        4488                 :
        4489        43659702 :         if (run->nfree == bin->nregs) {
        4490                 :                 /* Deallocate run. */
        4491         2734632 :                 if (run == bin->runcur)
        4492         2572218 :                         bin->runcur = NULL;
        4493          162414 :                 else if (bin->nregs != 1) {
        4494          324828 :                         size_t run_pageind = (((uintptr_t)run -
        4495          162414 :                             (uintptr_t)chunk)) >> pagesize_2pow;
        4496          162414 :                         arena_chunk_map_t *run_mapelm =
        4497          162414 :                             &chunk->map[run_pageind];
        4498                 :                         /*
        4499                 :                          * This block's conditional is necessary because if the
        4500                 :                          * run only contains one region, then it never gets
        4501                 :                          * inserted into the non-full runs tree.
        4502                 :                          */
        4503          162414 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
        4504                 :                             run_mapelm);
        4505          162414 :                         arena_run_tree_remove(&bin->runs, run_mapelm);
        4506                 :                 }
        4507                 : #ifdef MALLOC_DEBUG
        4508         2734632 :                 run->magic = 0;
        4509                 : #endif
        4510                 :                 VALGRIND_FREELIKE_BLOCK(run, 0);
        4511         2734632 :                 arena_run_dalloc(arena, run, true);
        4512                 : #ifdef MALLOC_STATS
        4513         2734632 :                 bin->stats.curruns--;
        4514                 : #endif
        4515        40925070 :         } else if (run->nfree == 1 && run != bin->runcur) {
        4516                 :                 /*
        4517                 :                  * Make sure that bin->runcur always refers to the lowest
        4518                 :                  * non-full run, if one exists.
        4519                 :                  */
        4520          300701 :                 if (bin->runcur == NULL)
        4521            5398 :                         bin->runcur = run;
        4522          295303 :                 else if ((uintptr_t)run < (uintptr_t)bin->runcur) {
        4523                 :                         /* Switch runcur. */
        4524          212684 :                         if (bin->runcur->nfree > 0) {
        4525          210677 :                                 arena_chunk_t *runcur_chunk =
        4526          210677 :                                     (arena_chunk_t*)CHUNK_ADDR2BASE(bin->runcur);
        4527          210677 :                                 size_t runcur_pageind =
        4528          421354 :                                     (((uintptr_t)bin->runcur -
        4529          210677 :                                     (uintptr_t)runcur_chunk)) >> pagesize_2pow;
        4530          210677 :                                 arena_chunk_map_t *runcur_mapelm =
        4531          210677 :                                     &runcur_chunk->map[runcur_pageind];
        4532                 :
        4533                 :                                 /* Insert runcur. */
        4534          210677 :                                 assert(arena_run_tree_search(&bin->runs,
        4535                 :                                     runcur_mapelm) == NULL);
        4536          210677 :                                 arena_run_tree_insert(&bin->runs,
        4537                 :                                     runcur_mapelm);
        4538                 :                         }
        4539          212684 :                         bin->runcur = run;
        4540                 :                 } else {
        4541          165238 :                         size_t run_pageind = (((uintptr_t)run -
        4542           82619 :                             (uintptr_t)chunk)) >> pagesize_2pow;
        4543           82619 :                         arena_chunk_map_t *run_mapelm =
        4544           82619 :                             &chunk->map[run_pageind];
        4545                 :
        4546           82619 :                         assert(arena_run_tree_search(&bin->runs, run_mapelm) ==
        4547                 :                             NULL);
        4548           82619 :                         arena_run_tree_insert(&bin->runs, run_mapelm);
        4549                 :                 }
        4550                 :         }
        4551                 : #ifdef MALLOC_STATS
        4552        43659702 :         arena->stats.allocated_small -= size;
        4553        43659702 :         arena->stats.ndalloc_small++;
        4554                 : #endif
        4555        43659702 : }
        4556                 :
        4557                 : static void
        4558          637622 : arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
        4559                 : {
        4560                 :         /* Large allocation. */
        4561          637622 :         malloc_spin_lock(&arena->lock);
        4562                 :
        4563                 : #ifdef MALLOC_FILL
        4564                 : #ifndef MALLOC_STATS
        4565                 :         if (opt_junk)
        4566                 : #endif
        4567                 : #endif
        4568                 :         {
        4569          637622 :                 size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >>
        4570                 :                     pagesize_2pow;
        4571          637622 :                 size_t size = chunk->map[pageind].bits & ~pagesize_mask;
        4572                 :
        4573                 : #ifdef MALLOC_FILL
        4574                 : #ifdef MALLOC_STATS
        4575          637622 :                 if (opt_junk)
        4576                 : #endif
        4577          637622 :                         memset(ptr, 0x5a, size);
        4578                 : #endif
        4579                 : #ifdef MALLOC_STATS
        4580          637622 :                 arena->stats.allocated_large -= size;
        4581                 : #endif
        4582                 :         }
        4583                 : #ifdef MALLOC_STATS
        4584          637622 :         arena->stats.ndalloc_large++;
        4585                 : #endif
        4586                 :
        4587          637622 :         arena_run_dalloc(arena, (arena_run_t *)ptr, true);
        4588          637622 :         malloc_spin_unlock(&arena->lock);
        4589          637622 : }
        4590                 :
        4591                 : static inline void
        4592        44297318 : arena_dalloc(void *ptr, size_t offset)
        4593                 : {
        4594                 :         arena_chunk_t *chunk;
        4595                 :         arena_t *arena;
        4596                 :         size_t pageind;
        4597                 :         arena_chunk_map_t *mapelm;
        4598                 :
        4599        44297318 :         assert(ptr != NULL);
        4600        44297318 :         assert(offset != 0);
        4601        44297318 :         assert(CHUNK_ADDR2OFFSET(ptr) == offset);
        4602                 :
        4603        44297318 :         chunk = (arena_chunk_t *) ((uintptr_t)ptr - offset);
        4604        44297318 :         arena = chunk->arena;
        4605        44297318 :         assert(arena != NULL);
        4606        44297318 :         assert(arena->magic == ARENA_MAGIC);
        4607                 :
        4608        44297318 :         pageind = offset >> pagesize_2pow;
        4609        44297318 :         mapelm = &chunk->map[pageind];
        4610        44297318 :         assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
        4611        44297318 :         if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
        4612                 :                 /* Small allocation. */
        4613        43659696 :                 malloc_spin_lock(&arena->lock);
        4614        43659702 :                 arena_dalloc_small(arena, chunk, ptr, mapelm);
        4615        43659702 :                 malloc_spin_unlock(&arena->lock);
        4616                 :         } else
        4617          637622 :                 arena_dalloc_large(arena, chunk, ptr);
        4618                 :         VALGRIND_FREELIKE_BLOCK(ptr, 0);
        4619        44297323 : }
        4620                 :
        4621                 : static inline void
        4622         2262557 : idalloc(void *ptr)
        4623                 : {
        4624                 :         size_t offset;
        4625                 :
        4626         2262557 :         assert(ptr != NULL);
        4627                 :
        4628         2262557 :         offset = CHUNK_ADDR2OFFSET(ptr);
        4629         2262557 :         if (offset != 0)
        4630         2260937 :                 arena_dalloc(ptr, offset);
        4631                 :         else
        4632            1620 :                 huge_dalloc(ptr);
        4633         2262557 : }
        4634                 :
        4635                 : static void
        4636             117 : arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
        4637                 :     size_t size, size_t oldsize)
        4638                 : {
        4639                 :
        4640             117 :         assert(size < oldsize);
        4641                 :
        4642                 :         /*
        4643                 :          * Shrink the run, and make trailing pages available for other
        4644                 :          * allocations.
        4645                 :          */
        4646                 : #ifdef MALLOC_BALANCE
        4647                 :         arena_lock_balance(arena);
        4648                 : #else
        4649             117 :         malloc_spin_lock(&arena->lock);
        4650                 : #endif
        4651             117 :         arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size,
        4652                 :             true);
        4653                 : #ifdef MALLOC_STATS
        4654             117 :         arena->stats.allocated_large -= oldsize - size;
        4655                 : #endif
        4656             117 :         malloc_spin_unlock(&arena->lock);
        4657             117 : }
        4658                 :
        4659                 : static bool
        4660           28113 : arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
        4661                 :     size_t size, size_t oldsize)
        4662                 : {
        4663           28113 :         size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> pagesize_2pow;
        4664           28113 :         size_t npages = oldsize >> pagesize_2pow;
        4665                 :
        4666           28113 :         assert(oldsize == (chunk->map[pageind].bits & ~pagesize_mask));
        4667                 :
        4668                 :         /* Try to extend the run. */
        4669           28113 :         assert(size > oldsize);
        4670                 : #ifdef MALLOC_BALANCE
        4671                 :         arena_lock_balance(arena);
        4672                 : #else
        4673           28113 :         malloc_spin_lock(&arena->lock);
        4674                 : #endif
        4675           51977 :         if (pageind + npages < chunk_npages && (chunk->map[pageind+npages].bits
        4676           44938 :             & CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[pageind+npages].bits &
        4677           10537 :             ~pagesize_mask) >= size - oldsize) {
        4678                 :                 /*
        4679                 :                  * The next run is available and sufficiently large.  Split the
        4680                 :                  * following run, then merge the first part with the existing
        4681                 :                  * allocation.
        4682                 :                  */
        4683           18236 :                 arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
        4684            9118 :                     ((pageind+npages) << pagesize_2pow)), size - oldsize, true,
        4685                 :                     false);
        4686                 :
        4687            9118 :                 chunk->map[pageind].bits = size | CHUNK_MAP_LARGE |
        4688                 :                     CHUNK_MAP_ALLOCATED;
        4689            9118 :                 chunk->map[pageind+npages].bits = CHUNK_MAP_LARGE |
        4690                 :                     CHUNK_MAP_ALLOCATED;
        4691                 :
        4692                 : #ifdef MALLOC_STATS
        4693            9118 :                 arena->stats.allocated_large += size - oldsize;
        4694                 : #endif
        4695            9118 :                 malloc_spin_unlock(&arena->lock);
        4696            9118 :                 return (false);
        4697                 :         }
        4698           18995 :         malloc_spin_unlock(&arena->lock);
        4699                 :
        4700           18995 :         return (true);
        4701                 : }
        4702                 :
        4703                 : /*
        4704                 :  * Try to resize a large allocation, in order to avoid copying.  This will
        4705                 :  * always fail if growing an object, and the following run is already in use.
        4706                 :  */
        4707                 : static bool
        4708           28338 : arena_ralloc_large(void *ptr, size_t size, size_t oldsize)
        4709                 : {
        4710                 :         size_t psize;
        4711                 :
        4712           28338 :         psize = PAGE_CEILING(size);
        4713           28338 :         if (psize == oldsize) {
        4714                 :                 /* Same size class. */
        4715                 : #ifdef MALLOC_FILL
        4716             108 :                 if (opt_junk && size < oldsize) {
        4717             108 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize -
        4718                 :                             size);
        4719                 :                 }
        4720                 : #endif
        4721             108 :                 return (false);
        4722                 :         } else {
        4723                 :                 arena_chunk_t *chunk;
        4724                 :                 arena_t *arena;
        4725                 :
        4726           28230 :                 chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
        4727           28230 :                 arena = chunk->arena;
        4728           28230 :                 assert(arena->magic == ARENA_MAGIC);
        4729                 :
        4730           28230 :                 if (psize < oldsize) {
        4731                 : #ifdef MALLOC_FILL
        4732                 :                         /* Fill before shrinking in order avoid a race. */
        4733             117 :                         if (opt_junk) {
        4734             117 :                                 memset((void *)((uintptr_t)ptr + size), 0x5a,
        4735                 :                                     oldsize - size);
        4736                 :                         }
        4737                 : #endif
        4738             117 :                         arena_ralloc_large_shrink(arena, chunk, ptr, psize,
        4739                 :                             oldsize);
        4740             117 :                         return (false);
        4741                 :                 } else {
        4742           28113 :                         bool ret = arena_ralloc_large_grow(arena, chunk, ptr,
        4743                 :                             psize, oldsize);
        4744                 : #ifdef MALLOC_FILL
        4745           28113 :                         if (ret == false && opt_zero) {
        4746               0 :                                 memset((void *)((uintptr_t)ptr + oldsize), 0,
        4747                 :                                     size - oldsize);
        4748                 :                         }
        4749                 : #endif
        4750           28113 :                         return (ret);
        4751                 :                 }
        4752                 :         }
        4753                 : }
        4754                 :
        4755                 : static void *
        4756         2299975 : arena_ralloc(void *ptr, size_t size, size_t oldsize)
        4757                 : {
        4758                 :         void *ret;
        4759                 :         size_t copysize;
        4760                 :
        4761                 :         /* Try to avoid moving the allocation. */
        4762         2299975 :         if (size < small_min) {
        4763          337330 :                 if (oldsize < small_min &&
        4764           83165 :                     ffs((int)(pow2_ceil(size) >> (TINY_MIN_2POW + 1)))
        4765           83165 :                     == ffs((int)(pow2_ceil(oldsize) >> (TINY_MIN_2POW + 1))))
        4766            9324 :                         goto IN_PLACE; /* Same size class. */
        4767         2045810 :         } else if (size <= small_max) {
        4768         3339456 :                 if (oldsize >= small_min && oldsize <= small_max &&
        4769         1653543 :                     (QUANTUM_CEILING(size) >> opt_quantum_2pow)
        4770         1653543 :                     == (QUANTUM_CEILING(oldsize) >> opt_quantum_2pow))
        4771           18432 :                         goto IN_PLACE; /* Same size class. */
        4772          359897 :         } else if (size <= bin_maxclass) {
        4773          393045 :                 if (oldsize > small_max && oldsize <= bin_maxclass &&
        4774           89419 :                     pow2_ceil(size) == pow2_ceil(oldsize))
        4775            2007 :                         goto IN_PLACE; /* Same size class. */
        4776           56271 :         } else if (oldsize > bin_maxclass && oldsize <= arena_maxclass) {
        4777           28338 :                 assert(size > bin_maxclass);
        4778           28338 :                 if (arena_ralloc_large(ptr, size, oldsize) == false)
        4779            9343 :                         return (ptr);
        4780                 :         }
        4781                 :
        4782                 :         /*
        4783                 :          * If we get here, then size and oldsize are different enough that we
        4784                 :          * need to move the object.  In that case, fall back to allocating new
        4785                 :          * space and copying.
        4786                 :          */
        4787         2260869 :         ret = arena_malloc(choose_arena(), size, false);
        4788         2260869 :         if (ret == NULL)
        4789               0 :                 return (NULL);
        4790                 :
        4791                 :         /* Junk/zero-filling were already done by arena_malloc(). */
        4792         2260869 :         copysize = (size < oldsize) ? size : oldsize;
        4793                 : #ifdef VM_COPY_MIN
        4794                 :         if (copysize >= VM_COPY_MIN)
        4795                 :                 pages_copy(ret, ptr, copysize);
        4796                 :         else
        4797                 : #endif
        4798         2260869 :                 memcpy(ret, ptr, copysize);
        4799         2260869 :         idalloc(ptr);
        4800         2260869 :         return (ret);
        4801                 : IN_PLACE:
        4802                 : #ifdef MALLOC_FILL
        4803           29763 :         if (opt_junk && size < oldsize)
        4804           20583 :                 memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size);
        4805            9180 :         else if (opt_zero && size > oldsize)
        4806               0 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize);
        4807                 : #endif
        4808           29763 :         return (ptr);
        4809                 : }
        4810                 :
        4811                 : static inline void *
        4812         2301663 : iralloc(void *ptr, size_t size)
        4813                 : {
        4814                 :         size_t oldsize;
        4815                 :
        4816         2301663 :         assert(ptr != NULL);
        4817         2301663 :         assert(size != 0);
        4818                 :
        4819         2301663 :         oldsize = isalloc(ptr);
        4820                 :
        4821                 : #ifndef MALLOC_VALGRIND
        4822         2301663 :         if (size <= arena_maxclass)
        4823         2299975 :                 return (arena_ralloc(ptr, size, oldsize));
        4824                 :         else
        4825            1688 :                 return (huge_ralloc(ptr, size, oldsize));
        4826                 : #else
        4827                 :         /*
        4828                 :          * Valgrind does not provide a public interface for modifying an
        4829                 :          * existing allocation, so use malloc/memcpy/free instead.
        4830                 :          */
        4831                 :         {
        4832                 :                 void *ret = imalloc(size);
        4833                 :                 if (ret != NULL) {
        4834                 :                         if (oldsize < size)
        4835                 :                             memcpy(ret, ptr, oldsize);
        4836                 :                         else
        4837                 :                             memcpy(ret, ptr, size);
        4838                 :                         idalloc(ptr);
        4839                 :                 }
        4840                 :                 return (ret);
        4841                 :         }
        4842                 : #endif
        4843                 : }
        4844                 :
        4845                 : static bool
        4846           18669 : arena_new(arena_t *arena)
        4847                 : {
        4848                 :         unsigned i;
        4849                 :         arena_bin_t *bin;
        4850                 :         size_t pow2_size, prev_run_size;
        4851                 :
        4852           18669 :         if (malloc_spin_init(&arena->lock))
        4853               0 :                 return (true);
        4854                 :
        4855                 : #ifdef MALLOC_STATS
        4856           18669 :         memset(&arena->stats, 0, sizeof(arena_stats_t));
        4857                 : #endif
        4858                 :
        4859                 :         /* Initialize chunks. */
        4860           18669 :         arena_chunk_tree_dirty_new(&arena->chunks_dirty);
        4861                 : #ifdef MALLOC_DOUBLE_PURGE
        4862                 :         LinkedList_Init(&arena->chunks_madvised);
        4863                 : #endif
        4864           18669 :         arena->spare = NULL;
        4865                 :
        4866           18669 :         arena->ndirty = 0;
        4867                 :
        4868           18669 :         arena_avail_tree_new(&arena->runs_avail);
        4869                 :
        4870                 : #ifdef MALLOC_BALANCE
        4871                 :         arena->contention = 0;
        4872                 : #endif
        4873                 :
        4874                 :         /* Initialize bins. */
        4875           18669 :         prev_run_size = pagesize;
        4876                 :
        4877                 :         /* (2^n)-spaced tiny bins. */
        4878           56007 :         for (i = 0; i < ntbins; i++) {
        4879           37338 :                 bin = &arena->bins[i];
        4880           37338 :                 bin->runcur = NULL;
        4881           37338 :                 arena_run_tree_new(&bin->runs);
        4882                 :
        4883           37338 :                 bin->reg_size = (1U << (TINY_MIN_2POW + i));
        4884                 :
        4885           37338 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
        4886                 :
        4887                 : #ifdef MALLOC_STATS
        4888           37338 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
        4889                 : #endif
        4890                 :         }
        4891                 :
        4892                 :         /* Quantum-spaced bins. */
        4893          616077 :         for (; i < ntbins + nqbins; i++) {
        4894          597408 :                 bin = &arena->bins[i];
        4895          597408 :                 bin->runcur = NULL;
        4896          597408 :                 arena_run_tree_new(&bin->runs);
        4897                 :
        4898          597408 :                 bin->reg_size = quantum * (i - ntbins + 1);
        4899                 :
        4900          597408 :                 pow2_size = pow2_ceil(quantum * (i - ntbins + 1));
        4901          597408 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
        4902                 :
        4903                 : #ifdef MALLOC_STATS
        4904          597408 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
        4905                 : #endif
        4906                 :         }
        4907                 :
        4908                 :         /* (2^n)-spaced sub-page bins. */
        4909           56007 :         for (; i < ntbins + nqbins + nsbins; i++) {
        4910           37338 :                 bin = &arena->bins[i];
        4911           37338 :                 bin->runcur = NULL;
        4912           37338 :                 arena_run_tree_new(&bin->runs);
        4913                 :
        4914           37338 :                 bin->reg_size = (small_max << (i - (ntbins + nqbins) + 1));
        4915                 :
        4916           37338 :                 prev_run_size = arena_bin_run_size_calc(bin, prev_run_size);
        4917                 :
        4918                 : #ifdef MALLOC_STATS
        4919           37338 :                 memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
        4920                 : #endif
        4921                 :         }
        4922                 :
        4923                 : #ifdef MALLOC_DEBUG
        4924           18669 :         arena->magic = ARENA_MAGIC;
        4925                 : #endif
        4926                 :
        4927           18669 :         return (false);
        4928                 : }
        4929                 :
        4930                 : /* Create a new arena and insert it into the arenas array at index ind. */
        4931                 : static arena_t *
        4932           18669 : arenas_extend(unsigned ind)
        4933                 : {
        4934                 :         arena_t *ret;
        4935                 :
        4936                 :         /* Allocate enough space for trailing bins. */
        4937           18669 :         ret = (arena_t *)base_alloc(sizeof(arena_t)
        4938           18669 :             + (sizeof(arena_bin_t) * (ntbins + nqbins + nsbins - 1)));
        4939           18669 :         if (ret != NULL && arena_new(ret) == false) {
        4940           18669 :                 arenas[ind] = ret;
        4941           18669 :                 return (ret);
        4942                 :         }
        4943                 :         /* Only reached if there is an OOM error. */
        4944                 :
        4945                 :         /*
        4946                 :          * OOM here is quite inconvenient to propagate, since dealing with it
        4947                 :          * would require a check for failure in the fast path.  Instead, punt
        4948                 :          * by using arenas[0].  In practice, this is an extremely unlikely
        4949                 :          * failure.
        4950                 :          */
        4951               0 :         _malloc_message(_getprogname(),
        4952                 :             ": (malloc) Error initializing arena\n", "", "");
        4953               0 :         if (opt_abort)
        4954               0 :                 abort();
        4955                 :
        4956               0 :         return (arenas[0]);
        4957                 : }
        4958                 :
        4959                 : /*
        4960                 :  * End arena.
        4961                 :  */
        4962                 : /******************************************************************************/
        4963                 : /*
        4964                 :  * Begin general internal functions.
        4965                 :  */
        4966                 :
        4967                 : static void *
        4968            4554 : huge_malloc(size_t size, bool zero)
        4969                 : {
        4970                 :         void *ret;
        4971                 :         size_t csize;
        4972                 :         size_t psize;
        4973                 :         extent_node_t *node;
        4974                 :
        4975                 :         /* Allocate one or more contiguous chunks for this request. */
        4976                 :
        4977            4554 :         csize = CHUNK_CEILING(size);
        4978            4554 :         if (csize == 0) {
        4979                 :                 /* size is large enough to cause size_t wrap-around. */
        4980               0 :                 return (NULL);
        4981                 :         }
        4982                 :
        4983                 :         /* Allocate an extent node with which to track the chunk. */
        4984            4554 :         node = base_node_alloc();
        4985            4554 :         if (node == NULL)
        4986               0 :                 return (NULL);
        4987                 :
        4988            4554 :         ret = chunk_alloc(csize, zero, true);
        4989            4554 :         if (ret == NULL) {
        4990              18 :                 base_node_dealloc(node);
        4991              18 :                 return (NULL);
        4992                 :         }
        4993                 :
        4994                 :         /* Insert node into huge. */
        4995            4536 :         node->addr = ret;
        4996            4536 :         psize = PAGE_CEILING(size);
        4997            4536 :         node->size = psize;
        4998                 :
        4999            4536 :         malloc_mutex_lock(&huge_mtx);
        5000            4536 :         extent_tree_ad_insert(&huge, node);
        5001                 : #ifdef MALLOC_STATS
        5002            4536 :         huge_nmalloc++;
        5003                 :
        5004                 :         /* Although we allocated space for csize bytes, we indicate that we've
        5005                 :          * allocated only psize bytes.
        5006                 :          *
        5007                 :          * If DECOMMIT is defined, this is a reasonable thing to do, since
        5008                 :          * we'll explicitly decommit the bytes in excess of psize.
        5009                 :          *
        5010                 :          * If DECOMMIT is not defined, then we're relying on the OS to be lazy
        5011                 :          * about how it allocates physical pages to mappings.  If we never
        5012                 :          * touch the pages in excess of psize, the OS won't allocate a physical
        5013                 :          * page, and we won't use more than psize bytes of physical memory.
        5014                 :          *
        5015                 :          * A correct program will only touch memory in excess of how much it
        5016                 :          * requested if it first calls malloc_usable_size and finds out how
        5017                 :          * much space it has to play with.  But because we set node->size =
        5018                 :          * psize above, malloc_usable_size will return psize, not csize, and
        5019                 :          * the program will (hopefully) never touch bytes in excess of psize.
        5020                 :          * Thus those bytes won't take up space in physical memory, and we can
        5021                 :          * reasonably claim we never "allocated" them in the first place. */
        5022            4536 :         huge_allocated += psize;
        5023                 : #endif
        5024            4536 :         malloc_mutex_unlock(&huge_mtx);
        5025                 :
        5026                 : #ifdef MALLOC_DECOMMIT
        5027                 :         if (csize - psize > 0)
        5028                 :                 pages_decommit((void *)((uintptr_t)ret + psize), csize - psize);
        5029                 : #endif
        5030                 :
        5031                 : #ifdef MALLOC_DECOMMIT
        5032                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, zero);
        5033                 : #else
        5034                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, csize, 0, zero);
        5035                 : #endif
        5036                 :
        5037                 : #ifdef MALLOC_FILL
        5038            4536 :         if (zero == false) {
        5039            4518 :                 if (opt_junk)
        5040                 : #  ifdef MALLOC_DECOMMIT
        5041                 :                         memset(ret, 0xa5, psize);
        5042                 : #  else
        5043            4518 :                         memset(ret, 0xa5, csize);
        5044                 : #  endif
        5045               0 :                 else if (opt_zero)
        5046                 : #  ifdef MALLOC_DECOMMIT
        5047                 :                         memset(ret, 0, psize);
        5048                 : #  else
        5049               0 :                         memset(ret, 0, csize);
        5050                 : #  endif
        5051                 :         }
        5052                 : #endif
        5053                 :
        5054            4536 :         return (ret);
        5055                 : }
        5056                 :
        5057                 : /* Only handles large allocations that require more than chunk alignment. */
        5058                 : static void *
        5059               0 : huge_palloc(size_t alignment, size_t size)
        5060                 : {
        5061                 :         void *ret;
        5062                 :         size_t alloc_size, chunk_size, offset;
        5063                 :         size_t psize;
        5064                 :         extent_node_t *node;
        5065                 :         int pfd;
        5066                 :
        5067                 :         /*
        5068                 :          * This allocation requires alignment that is even larger than chunk
        5069                 :          * alignment.  This means that huge_malloc() isn't good enough.
        5070                 :          *
        5071                 :          * Allocate almost twice as many chunks as are demanded by the size or
        5072                 :          * alignment, in order to assure the alignment can be achieved, then
        5073                 :          * unmap leading and trailing chunks.
        5074                 :          */
        5075               0 :         assert(alignment >= chunksize);
        5076                 :
        5077               0 :         chunk_size = CHUNK_CEILING(size);
        5078                 :
        5079               0 :         if (size >= alignment)
        5080               0 :                 alloc_size = chunk_size + alignment - chunksize;
        5081                 :         else
        5082               0 :                 alloc_size = (alignment << 1) - chunksize;
        5083                 :
        5084                 :         /* Allocate an extent node with which to track the chunk. */
        5085               0 :         node = base_node_alloc();
        5086               0 :         if (node == NULL)
        5087               0 :                 return (NULL);
        5088                 :
        5089                 :         /*
        5090                 :          * Windows requires that there be a 1:1 mapping between VM
        5091                 :          * allocation/deallocation operations.  Therefore, take care here to
        5092                 :          * acquire the final result via one mapping operation.
        5093                 :          *
        5094                 :          * The MALLOC_PAGEFILE code also benefits from this mapping algorithm,
        5095                 :          * since it reduces the number of page files.
        5096                 :          */
        5097                 : #ifdef MALLOC_PAGEFILE
        5098                 :         if (opt_pagefile) {
        5099                 :                 pfd = pagefile_init(size);
        5100                 :                 if (pfd == -1)
        5101                 :                         return (NULL);
        5102                 :         } else
        5103                 : #endif
        5104               0 :                 pfd = -1;
        5105                 : #ifdef JEMALLOC_USES_MAP_ALIGN
        5106                 :                 ret = pages_map_align(chunk_size, pfd, alignment);
        5107                 : #else
        5108                 :         do {
        5109                 :                 void *over;
        5110                 :
        5111               0 :                 over = chunk_alloc(alloc_size, false, false);
        5112               0 :                 if (over == NULL) {
        5113               0 :                         base_node_dealloc(node);
        5114               0 :                         ret = NULL;
        5115               0 :                         goto RETURN;
        5116                 :                 }
        5117                 :
        5118               0 :                 offset = (uintptr_t)over & (alignment - 1);
        5119               0 :                 assert((offset & chunksize_mask) == 0);
        5120               0 :                 assert(offset < alloc_size);
        5121               0 :                 ret = (void *)((uintptr_t)over + offset);
        5122               0 :                 chunk_dealloc(over, alloc_size);
        5123               0 :                 ret = pages_map(ret, chunk_size, pfd);
        5124                 :                 /*
        5125                 :                  * Failure here indicates a race with another thread, so try
        5126                 :                  * again.
        5127                 :                  */
        5128               0 :         } while (ret == NULL);
        5129                 : #endif
        5130                 :         /* Insert node into huge. */
        5131               0 :         node->addr = ret;
        5132               0 :         psize = PAGE_CEILING(size);
        5133               0 :         node->size = psize;
        5134                 :
        5135               0 :         malloc_mutex_lock(&huge_mtx);
        5136               0 :         extent_tree_ad_insert(&huge, node);
        5137                 : #ifdef MALLOC_STATS
        5138               0 :         huge_nmalloc++;
        5139                 :         /* See note in huge_alloc() for why huge_allocated += psize is correct
        5140                 :          * here even when DECOMMIT is not defined. */
        5141               0 :         huge_allocated += psize;
        5142                 : #endif
        5143               0 :         malloc_mutex_unlock(&huge_mtx);
        5144                 :
        5145                 : #ifdef MALLOC_DECOMMIT
        5146                 :         if (chunk_size - psize > 0) {
        5147                 :                 pages_decommit((void *)((uintptr_t)ret + psize),
        5148                 :                     chunk_size - psize);
        5149                 :         }
        5150                 : #endif
        5151                 :
        5152                 : #ifdef MALLOC_DECOMMIT
        5153                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, psize, 0, false);
        5154                 : #else
        5155                 :         VALGRIND_MALLOCLIKE_BLOCK(ret, chunk_size, 0, false);
        5156                 : #endif
        5157                 :
        5158                 : #ifdef MALLOC_FILL
        5159               0 :         if (opt_junk)
        5160                 : #  ifdef MALLOC_DECOMMIT
        5161                 :                 memset(ret, 0xa5, psize);
        5162                 : #  else
        5163               0 :                 memset(ret, 0xa5, chunk_size);
        5164                 : #  endif
        5165               0 :         else if (opt_zero)
        5166                 : #  ifdef MALLOC_DECOMMIT
        5167                 :                 memset(ret, 0, psize);
        5168                 : #  else
        5169               0 :                 memset(ret, 0, chunk_size);
        5170                 : #  endif
        5171                 : #endif
        5172                 :
        5173                 : RETURN:
        5174                 : #ifdef MALLOC_PAGEFILE
        5175                 :         if (pfd != -1)
        5176                 :                 pagefile_close(pfd);
        5177                 : #endif
        5178               0 :         return (ret);
        5179                 : }
        5180                 :
        5181                 : static void *
        5182            1688 : huge_ralloc(void *ptr, size_t size, size_t oldsize)
        5183                 : {
        5184                 :         void *ret;
        5185                 :         size_t copysize;
        5186                 :
        5187                 :         /* Avoid moving the allocation if the size class would not change. */
        5188                 :
        5189            3191 :         if (oldsize > arena_maxclass &&
        5190            1503 :             CHUNK_CEILING(size) == CHUNK_CEILING(oldsize)) {
        5191               0 :                 size_t psize = PAGE_CEILING(size);
        5192                 : #ifdef MALLOC_FILL
        5193               0 :                 if (opt_junk && size < oldsize) {
        5194               0 :                         memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize
        5195                 :                             - size);
        5196                 :                 }
        5197                 : #endif
        5198                 : #ifdef MALLOC_DECOMMIT
        5199                 :                 if (psize < oldsize) {
        5200                 :                         extent_node_t *node, key;
        5201                 :
        5202                 :                         pages_decommit((void *)((uintptr_t)ptr + psize),
        5203                 :                             oldsize - psize);
        5204                 :
        5205                 :                         /* Update recorded size. */
        5206                 :                         malloc_mutex_lock(&huge_mtx);
        5207                 :                         key.addr = __DECONST(void *, ptr);
        5208                 :                         node = extent_tree_ad_search(&huge, &key);
        5209                 :                         assert(node != NULL);
        5210                 :                         assert(node->size == oldsize);
        5211                 : #  ifdef MALLOC_STATS
        5212                 :                         huge_allocated -= oldsize - psize;
        5213                 : #  endif
        5214                 :                         node->size = psize;
        5215                 :                         malloc_mutex_unlock(&huge_mtx);
        5216                 :                 } else if (psize > oldsize) {
        5217                 :                         pages_commit((void *)((uintptr_t)ptr + oldsize),
        5218                 :                             psize - oldsize);
        5219                 :                 }
        5220                 : #endif
        5221                 :
        5222                 :                 /* Although we don't have to commit or decommit anything if
        5223                 :                  * DECOMMIT is not defined and the size class didn't change, we
        5224                 :                  * do need to update the recorded size if the size increased,
        5225                 :                  * so malloc_usable_size doesn't return a value smaller than
        5226                 :                  * what was requested via realloc(). */
        5227                 :
        5228               0 :                 if (psize > oldsize) {
        5229                 :                         /* Update recorded size. */
        5230                 :                         extent_node_t *node, key;
        5231               0 :                         malloc_mutex_lock(&huge_mtx);
        5232               0 :                         key.addr = __DECONST(void *, ptr);
        5233               0 :                         node = extent_tree_ad_search(&huge, &key);
        5234               0 :                         assert(node != NULL);
        5235               0 :                         assert(node->size == oldsize);
        5236                 : #  ifdef MALLOC_STATS
        5237               0 :                         huge_allocated += psize - oldsize;
        5238                 : #  endif
        5239               0 :                         node->size = psize;
        5240               0 :                         malloc_mutex_unlock(&huge_mtx);
        5241                 :                 }
        5242                 :
        5243                 : #ifdef MALLOC_FILL
        5244               0 :                 if (opt_zero && size > oldsize) {
        5245               0 :                         memset((void *)((uintptr_t)ptr + oldsize), 0, size
        5246                 :                             - oldsize);
        5247                 :                 }
        5248                 : #endif
        5249               0 :                 return (ptr);
        5250                 :         }
        5251                 :
        5252                 :         /*
        5253                 :          * If we get here, then size and oldsize are different enough that we
        5254                 :          * need to use a different size class.  In that case, fall back to
        5255                 :          * allocating new space and copying.
        5256                 :          */
        5257            1688 :         ret = huge_malloc(size, false);
        5258            1688 :         if (ret == NULL)
        5259               0 :                 return (NULL);
        5260                 :
        5261            1688 :         copysize = (size < oldsize) ? size : oldsize;
        5262                 : #ifdef VM_COPY_MIN
        5263                 :         if (copysize >= VM_COPY_MIN)
        5264                 :                 pages_copy(ret, ptr, copysize);
        5265                 :         else
        5266                 : #endif
        5267            1688 :                 memcpy(ret, ptr, copysize);
        5268            1688 :         idalloc(ptr);
        5269            1688 :         return (ret);
        5270                 : }
        5271                 :
        5272                 : static void
        5273            4536 : huge_dalloc(void *ptr)
        5274                 : {
        5275                 :         extent_node_t *node, key;
        5276                 :
        5277            4536 :         malloc_mutex_lock(&huge_mtx);
        5278                 :
        5279                 :         /* Extract from tree of huge allocations. */
        5280            4536 :         key.addr = ptr;
        5281            4536 :         node = extent_tree_ad_search(&huge, &key);
        5282            4536 :         assert(node != NULL);
        5283            4536 :         assert(node->addr == ptr);
        5284            4536 :         extent_tree_ad_remove(&huge, node);
        5285                 :
        5286                 : #ifdef MALLOC_STATS
        5287            4536 :         huge_ndalloc++;
        5288            4536 :         huge_allocated -= node->size;
        5289                 : #endif
        5290                 :
        5291            4536 :         malloc_mutex_unlock(&huge_mtx);
        5292                 :
        5293                 :         /* Unmap chunk. */
        5294                 : #ifdef MALLOC_FILL
        5295            4536 :         if (opt_junk)
        5296            4536 :                 memset(node->addr, 0x5a, node->size);
        5297                 : #endif
        5298            4536 :         chunk_dealloc(node->addr, CHUNK_CEILING(node->size));
        5299                 :         VALGRIND_FREELIKE_BLOCK(node->addr, 0);
        5300                 :
        5301            4536 :         base_node_dealloc(node);
        5302            4536 : }
        5303                 :
        5304                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        5305                 : #ifdef MOZ_MEMORY_BSD
        5306                 : static inline unsigned
        5307                 : malloc_ncpus(void)
        5308                 : {
        5309                 :         unsigned ret;
        5310                 :         int mib[2];
        5311                 :         size_t len;
        5312                 :
        5313                 :         mib[0] = CTL_HW;
        5314                 :         mib[1] = HW_NCPU;
        5315                 :         len = sizeof(ret);
        5316                 :         if (sysctl(mib, 2, &ret, &len, (void *) 0, 0) == -1) {
        5317                 :                 /* Error. */
        5318                 :                 return (1);
        5319                 :         }
        5320                 :
        5321                 :         return (ret);
        5322                 : }
        5323                 : #elif (defined(MOZ_MEMORY_LINUX))
        5324                 : #include <fcntl.h>
        5325                 :
        5326                 : static inline unsigned
        5327                 : malloc_ncpus(void)
        5328                 : {
        5329                 :         unsigned ret;
        5330                 :         int fd, nread, column;
        5331                 :         char buf[1024];
        5332                 :         static const char matchstr[] = "processor\t:";
        5333                 :         int i;
        5334                 :
        5335                 :         /*
        5336                 :          * sysconf(3) would be the preferred method for determining the number
        5337                 :          * of CPUs, but it uses malloc internally, which causes untennable
        5338                 :          * recursion during malloc initialization.
        5339                 :          */
        5340                 :         fd = open("/proc/cpuinfo", O_RDONLY);
        5341                 :         if (fd == -1)
        5342                 :                 return (1); /* Error. */
        5343                 :         /*
        5344                 :          * Count the number of occurrences of matchstr at the beginnings of
        5345                 :          * lines.  This treats hyperthreaded CPUs as multiple processors.
        5346                 :          */
        5347                 :         column = 0;
        5348                 :         ret = 0;
        5349                 :         while (true) {
        5350                 :                 nread = read(fd, &buf, sizeof(buf));
        5351                 :                 if (nread <= 0)
        5352                 :                         break; /* EOF or error. */
        5353                 :                 for (i = 0;i < nread;i++) {
        5354                 :                         char c = buf[i];
        5355                 :                         if (c == '\n')
        5356                 :                                 column = 0;
        5357                 :                         else if (column != -1) {
        5358                 :                                 if (c == matchstr[column]) {
        5359                 :                                         column++;
        5360                 :                                         if (column == sizeof(matchstr) - 1) {
        5361                 :                                                 column = -1;
        5362                 :                                                 ret++;
        5363                 :                                         }
        5364                 :                                 } else
        5365                 :                                         column = -1;
        5366                 :                         }
        5367                 :                 }
        5368                 :         }
        5369                 :
        5370                 :         if (ret == 0)
        5371                 :                 ret = 1; /* Something went wrong in the parser. */
        5372                 :         close(fd);
        5373                 :
        5374                 :         return (ret);
        5375                 : }
        5376                 : #elif (defined(MOZ_MEMORY_DARWIN))
        5377                 : #include <mach/mach_init.h>
        5378                 : #include <mach/mach_host.h>
        5379                 :
        5380                 : static inline unsigned
        5381                 : malloc_ncpus(void)
        5382                 : {
        5383                 :         kern_return_t error;
        5384                 :         natural_t n;
        5385                 :         processor_info_array_t pinfo;
        5386                 :         mach_msg_type_number_t pinfocnt;
        5387                 :
        5388                 :         error = host_processor_info(mach_host_self(), PROCESSOR_BASIC_INFO,
        5389                 :                                     &n, &pinfo, &pinfocnt);
        5390                 :         if (error != KERN_SUCCESS)
        5391                 :                 return (1); /* Error. */
        5392                 :         else
        5393                 :                 return (n);
        5394                 : }
        5395                 : #elif (defined(MOZ_MEMORY_SOLARIS))
        5396                 :
        5397                 : static inline unsigned
        5398                 : malloc_ncpus(void)
        5399                 : {
        5400                 :         return sysconf(_SC_NPROCESSORS_ONLN);
        5401                 : }
        5402                 : #else
        5403                 : static inline unsigned
        5404                 : malloc_ncpus(void)
        5405                 : {
        5406                 :
        5407                 :         /*
        5408                 :          * We lack a way to determine the number of CPUs on this platform, so
        5409                 :          * assume 1 CPU.
        5410                 :          */
        5411                 :         return (1);
        5412                 : }
        5413                 : #endif
        5414                 : #endif
        5415                 :
        5416                 : static void
        5417               0 : malloc_print_stats(void)
        5418                 : {
        5419                 :
        5420               0 :         if (opt_print_stats) {
        5421                 :                 char s[UMAX2S_BUFSIZE];
        5422               0 :                 _malloc_message("___ Begin malloc statistics ___\n", "", "",
        5423                 :                     "");
        5424               0 :                 _malloc_message("Assertions ",
        5425                 : #ifdef NDEBUG
        5426                 :                     "disabled",
        5427                 : #else
        5428                 :                     "enabled",
        5429                 : #endif
        5430                 :                     "\n", "");
        5431               0 :                 _malloc_message("Boolean MALLOC_OPTIONS: ",
        5432                 :                     opt_abort ? "A" : "a", "", "");
        5433                 : #ifdef MALLOC_FILL
        5434               0 :                 _malloc_message(opt_junk ? "J" : "j", "", "", "");
        5435                 : #endif
        5436                 : #ifdef MALLOC_PAGEFILE
        5437                 :                 _malloc_message(opt_pagefile ? "o" : "O", "", "", "");
        5438                 : #endif
        5439               0 :                 _malloc_message("P", "", "", "");
        5440                 : #ifdef MALLOC_UTRACE
        5441               0 :                 _malloc_message(opt_utrace ? "U" : "u", "", "", "");
        5442                 : #endif
        5443                 : #ifdef MALLOC_SYSV
        5444               0 :                 _malloc_message(opt_sysv ? "V" : "v", "", "", "");
        5445                 : #endif
        5446                 : #ifdef MALLOC_XMALLOC
        5447               0 :                 _malloc_message(opt_xmalloc ? "X" : "x", "", "", "");
        5448                 : #endif
        5449                 : #ifdef MALLOC_FILL
        5450               0 :                 _malloc_message(opt_zero ? "Z" : "z", "", "", "");
        5451                 : #endif
        5452               0 :                 _malloc_message("\n", "", "", "");
        5453                 :
        5454                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        5455                 :                 _malloc_message("CPUs: ", umax2s(ncpus, 10, s), "\n", "");
        5456                 : #endif
        5457               0 :                 _malloc_message("Max arenas: ", umax2s(narenas, 10, s), "\n",
        5458                 :                     "");
        5459                 : #ifdef MALLOC_BALANCE
        5460                 :                 _malloc_message("Arena balance threshold: ",
        5461                 :                     umax2s(opt_balance_threshold, 10, s), "\n", "");
        5462                 : #endif
        5463               0 :                 _malloc_message("Pointer size: ", umax2s(sizeof(void *), 10, s),
        5464                 :                     "\n", "");
        5465               0 :                 _malloc_message("Quantum size: ", umax2s(quantum, 10, s), "\n",
        5466                 :                     "");
        5467               0 :                 _malloc_message("Max small size: ", umax2s(small_max, 10, s),
        5468                 :                     "\n", "");
        5469               0 :                 _malloc_message("Max dirty pages per arena: ",
        5470               0 :                     umax2s(opt_dirty_max, 10, s), "\n", "");
        5471                 :
        5472               0 :                 _malloc_message("Chunk size: ", umax2s(chunksize, 10, s), "",
        5473                 :                     "");
        5474               0 :                 _malloc_message(" (2^", umax2s(opt_chunk_2pow, 10, s), ")\n",
        5475                 :                     "");
        5476                 :
        5477                 : #ifdef MALLOC_STATS
        5478                 :                 {
        5479                 :                         size_t allocated, mapped;
        5480                 : #ifdef MALLOC_BALANCE
        5481                 :                         uint64_t nbalance = 0;
        5482                 : #endif
        5483                 :                         unsigned i;
        5484                 :                         arena_t *arena;
        5485                 :
        5486                 :                         /* Calculate and print allocated/mapped stats. */
        5487                 :
        5488                 :                         /* arenas. */
        5489               0 :                         for (i = 0, allocated = 0; i < narenas; i++) {
        5490               0 :                                 if (arenas[i] != NULL) {
        5491               0 :                                         malloc_spin_lock(&arenas[i]->lock);
        5492               0 :                                         allocated +=
        5493               0 :                                             arenas[i]->stats.allocated_small;
        5494               0 :                                         allocated +=
        5495               0 :                                             arenas[i]->stats.allocated_large;
        5496                 : #ifdef MALLOC_BALANCE
        5497                 :                                         nbalance += arenas[i]->stats.nbalance;
        5498                 : #endif
        5499               0 :                                         malloc_spin_unlock(&arenas[i]->lock);
        5500                 :                                 }
        5501                 :                         }
        5502                 :
        5503                 :                         /* huge/base. */
        5504               0 :                         malloc_mutex_lock(&huge_mtx);
        5505               0 :                         allocated += huge_allocated;
        5506               0 :                         mapped = stats_chunks.curchunks * chunksize;
        5507               0 :                         malloc_mutex_unlock(&huge_mtx);
        5508                 :
        5509               0 :                         malloc_mutex_lock(&base_mtx);
        5510               0 :                         mapped += base_mapped;
        5511               0 :                         malloc_mutex_unlock(&base_mtx);
        5512                 :
        5513                 : #ifdef MOZ_MEMORY_WINDOWS
        5514                 :                         malloc_printf("Allocated: %lu, mapped: %lu\n",
        5515                 :                             allocated, mapped);
        5516                 : #else
        5517               0 :                         malloc_printf("Allocated: %zu, mapped: %zu\n",
        5518                 :                             allocated, mapped);
        5519                 : #endif
        5520                 :
        5521                 : #ifdef MALLOC_BALANCE
        5522                 :                         malloc_printf("Arena balance reassignments: %llu\n",
        5523                 :                             nbalance);
        5524                 : #endif
        5525                 :
        5526                 :                         /* Print chunk stats. */
        5527                 :                         {
        5528                 :                                 chunk_stats_t chunks_stats;
        5529                 :
        5530               0 :                                 malloc_mutex_lock(&huge_mtx);
        5531               0 :                                 chunks_stats = stats_chunks;
        5532               0 :                                 malloc_mutex_unlock(&huge_mtx);
        5533                 :
        5534               0 :                                 malloc_printf("chunks: nchunks   "
        5535                 :                                     "highchunks    curchunks\n");
        5536               0 :                                 malloc_printf("  %13llu%13lu%13lu\n",
        5537                 :                                     chunks_stats.nchunks,
        5538                 :                                     chunks_stats.highchunks,
        5539                 :                                     chunks_stats.curchunks);
        5540                 :                         }
        5541                 :
        5542                 :                         /* Print chunk stats. */
        5543               0 :                         malloc_printf(
        5544                 :                             "huge: nmalloc      ndalloc    allocated\n");
        5545                 : #ifdef MOZ_MEMORY_WINDOWS
        5546                 :                         malloc_printf(" %12llu %12llu %12lu\n",
        5547                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
        5548                 : #else
        5549               0 :                         malloc_printf(" %12llu %12llu %12zu\n",
        5550                 :                             huge_nmalloc, huge_ndalloc, huge_allocated);
        5551                 : #endif
        5552                 :                         /* Print stats for each arena. */
        5553               0 :                         for (i = 0; i < narenas; i++) {
        5554               0 :                                 arena = arenas[i];
        5555               0 :                                 if (arena != NULL) {
        5556               0 :                                         malloc_printf(
        5557                 :                                             "\narenas[%u]:\n", i);
        5558               0 :                                         malloc_spin_lock(&arena->lock);
        5559               0 :                                         stats_print(arena);
        5560               0 :                                         malloc_spin_unlock(&arena->lock);
        5561                 :                                 }
        5562                 :                         }
        5563                 :                 }
        5564                 : #endif /* #ifdef MALLOC_STATS */
        5565               0 :                 _malloc_message("--- End malloc statistics ---\n", "", "", "");
        5566                 :         }
        5567               0 : }
        5568                 :
        5569                 : /*
        5570                 :  * FreeBSD's pthreads implementation calls malloc(3), so the malloc
        5571                 :  * implementation has to take pains to avoid infinite recursion during
        5572                 :  * initialization.
        5573                 :  */
        5574                 : #if (defined(MOZ_MEMORY_WINDOWS) || defined(MOZ_MEMORY_DARWIN))
        5575                 : #define malloc_init() false
        5576                 : #else
        5577                 : static inline bool
        5578        44114699 : malloc_init(void)
        5579                 : {
        5580                 :
        5581        44114699 :         if (malloc_initialized == false)
        5582           18669 :                 return (malloc_init_hard());
        5583                 :
        5584        44096030 :         return (false);
        5585                 : }
        5586                 : #endif
        5587                 :
        5588                 : #if !defined(MOZ_MEMORY_WINDOWS)
        5589                 : static
        5590                 : #endif
        5591                 : bool
        5592           18669 : malloc_init_hard(void)
        5593                 : {
        5594                 :         unsigned i;
        5595                 :         char buf[PATH_MAX + 1];
        5596                 :         const char *opts;
        5597                 :         long result;
        5598                 : #ifndef MOZ_MEMORY_WINDOWS
        5599                 :         int linklen;
        5600                 : #endif
        5601                 : #ifdef MOZ_MEMORY_DARWIN
        5602                 :     malloc_zone_t* default_zone;
        5603                 : #endif
        5604                 :
        5605                 : #ifndef MOZ_MEMORY_WINDOWS
        5606           18669 :         malloc_mutex_lock(&init_lock);
        5607                 : #endif
        5608                 :
        5609           18669 :         if (malloc_initialized) {
        5610                 :                 /*
        5611                 :                  * Another thread initialized the allocator before this one
        5612                 :                  * acquired init_lock.
        5613                 :                  */
        5614                 : #ifndef MOZ_MEMORY_WINDOWS
        5615               0 :                 malloc_mutex_unlock(&init_lock);
        5616                 : #endif
        5617               0 :                 return (false);
        5618                 :         }
        5619                 :
        5620                 : #ifdef MOZ_MEMORY_WINDOWS
        5621                 :         /* get a thread local storage index */
        5622                 :         tlsIndex = TlsAlloc();
        5623                 : #endif
        5624                 :
        5625                 :         /* Get page size and number of CPUs */
        5626                 : #ifdef MOZ_MEMORY_WINDOWS
        5627                 :         {
        5628                 :                 SYSTEM_INFO info;
        5629                 :
        5630                 :                 GetSystemInfo(&info);
        5631                 :                 result = info.dwPageSize;
        5632                 :
        5633                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        5634                 :                 ncpus = info.dwNumberOfProcessors;
        5635                 : #endif
        5636                 :         }
        5637                 : #else
        5638                 : #ifndef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        5639                 :         ncpus = malloc_ncpus();
        5640                 : #endif
        5641                 :
        5642           18669 :         result = sysconf(_SC_PAGESIZE);
        5643           18669 :         assert(result != -1);
        5644                 : #endif
        5645                 :
        5646                 :         /* We assume that the page size is a power of 2. */
        5647           18669 :         assert(((result - 1) & result) == 0);
        5648                 : #ifdef MALLOC_STATIC_SIZES
        5649           18669 :         if (pagesize % (size_t) result) {
        5650               0 :                 _malloc_message(_getprogname(),
        5651                 :                                 "Compile-time page size does not divide the runtime one.\n",
        5652                 :                                 "", "");
        5653               0 :                 abort();
        5654                 :         }
        5655                 : #else
        5656                 :         pagesize = (size_t) result;
        5657                 :         pagesize_mask = (size_t) result - 1;
        5658                 :         pagesize_2pow = ffs((int)result) - 1;
        5659                 : #endif
        5660                 :
        5661                 : #ifdef MALLOC_PAGEFILE
        5662                 :         /*
        5663                 :          * Determine where to create page files.  It is insufficient to
        5664                 :          * unconditionally use P_tmpdir (typically "/tmp"), since for some
        5665                 :          * operating systems /tmp is a separate filesystem that is rather small.
        5666                 :          * Therefore prefer, in order, the following locations:
        5667                 :          *
        5668                 :          * 1) MALLOC_TMPDIR
        5669                 :          * 2) TMPDIR
        5670                 :          * 3) P_tmpdir
        5671                 :          */
        5672                 :         {
        5673                 :                 char *s;
        5674                 :                 size_t slen;
        5675                 :                 static const char suffix[] = "/jemalloc.XXXXXX";
        5676                 :
        5677                 :                 if ((s = getenv("MALLOC_TMPDIR")) == NULL && (s =
        5678                 :                     getenv("TMPDIR")) == NULL)
        5679                 :                         s = P_tmpdir;
        5680                 :                 slen = strlen(s);
        5681                 :                 if (slen + sizeof(suffix) > sizeof(pagefile_templ)) {
        5682                 :                         _malloc_message(_getprogname(),
        5683                 :                             ": (malloc) Page file path too long\n",
        5684                 :                             "", "");
        5685                 :                         abort();
        5686                 :                 }
        5687                 :                 memcpy(pagefile_templ, s, slen);
        5688                 :                 memcpy(&pagefile_templ[slen], suffix, sizeof(suffix));
        5689                 :         }
        5690                 : #endif
        5691                 :
        5692           74676 :         for (i = 0; i < 3; i++) {
        5693                 :                 unsigned j;
        5694                 :
        5695                 :                 /* Get runtime configuration. */
        5696           56007 :                 switch (i) {
        5697                 :                 case 0:
        5698                 : #ifndef MOZ_MEMORY_WINDOWS
        5699           18669 :                         if ((linklen = readlink("/etc/malloc.conf", buf,
        5700                 :                                                 sizeof(buf) - 1)) != -1) {
        5701                 :                                 /*
        5702                 :                                  * Use the contents of the "/etc/malloc.conf"
        5703                 :                                  * symbolic link's name.
        5704                 :                                  */
        5705               0 :                                 buf[linklen] = '\0';
        5706               0 :                                 opts = buf;
        5707                 :                         } else
        5708                 : #endif
        5709                 :                         {
        5710                 :                                 /* No configuration specified. */
        5711           18669 :                                 buf[0] = '\0';
        5712           18669 :                                 opts = buf;
        5713                 :                         }
        5714           18669 :                         break;
        5715                 :                 case 1:
        5716           18669 :                         if (issetugid() == 0 && (opts =
        5717                 :                             getenv("MALLOC_OPTIONS")) != NULL) {
        5718                 :                                 /*
        5719                 :                                  * Do nothing; opts is already initialized to
        5720                 :                                  * the value of the MALLOC_OPTIONS environment
        5721                 :                                  * variable.
        5722                 :                                  */
        5723                 :                         } else {
        5724                 :                                 /* No configuration specified. */
        5725           18669 :                                 buf[0] = '\0';
        5726           18669 :                                 opts = buf;
        5727                 :                         }
        5728           18669 :                         break;
        5729                 :                 case 2:
        5730           18669 :                         if (_malloc_options != NULL) {
        5731                 :                                 /*
        5732                 :                                  * Use options that were compiled into the
        5733                 :                                  * program.
        5734                 :                                  */
        5735               0 :                                 opts = _malloc_options;
        5736                 :                         } else {
        5737                 :                                 /* No configuration specified. */
        5738           18669 :                                 buf[0] = '\0';
        5739           18669 :                                 opts = buf;
        5740                 :                         }
        5741           18669 :                         break;
        5742                 :                 default:
        5743                 :                         /* NOTREACHED */
        5744               0 :                         buf[0] = '\0';
        5745               0 :                         opts = buf;
        5746               0 :                         assert(false);
        5747                 :                 }
        5748                 :
        5749           56007 :                 for (j = 0; opts[j] != '\0'; j++) {
        5750                 :                         unsigned k, nreps;
        5751                 :                         bool nseen;
        5752                 :
        5753                 :                         /* Parse repetition count, if any. */
        5754               0 :                         for (nreps = 0, nseen = false;; j++, nseen = true) {
        5755               0 :                                 switch (opts[j]) {
        5756                 :                                         case '0': case '1': case '2': case '3':
        5757                 :                                         case '4': case '5': case '6': case '7':
        5758                 :                                         case '8': case '9':
        5759               0 :                                                 nreps *= 10;
        5760               0 :                                                 nreps += opts[j] - '0';
        5761                 :                                                 break;
        5762                 :                                         default:
        5763               0 :                                                 goto MALLOC_OUT;
        5764                 :                                 }
        5765               0 :                         }
        5766                 : MALLOC_OUT:
        5767               0 :                         if (nseen == false)
        5768               0 :                                 nreps = 1;
        5769                 :
        5770               0 :                         for (k = 0; k < nreps; k++) {
        5771               0 :                                 switch (opts[j]) {
        5772                 :                                 case 'a':
        5773               0 :                                         opt_abort = false;
        5774               0 :                                         break;
        5775                 :                                 case 'A':
        5776               0 :                                         opt_abort = true;
        5777               0 :                                         break;
        5778                 :                                 case 'b':
        5779                 : #ifdef MALLOC_BALANCE
        5780                 :                                         opt_balance_threshold >>= 1;
        5781                 : #endif
        5782               0 :                                         break;
        5783                 :                                 case 'B':
        5784                 : #ifdef MALLOC_BALANCE
        5785                 :                                         if (opt_balance_threshold == 0)
        5786                 :                                                 opt_balance_threshold = 1;
        5787                 :                                         else if ((opt_balance_threshold << 1)
        5788                 :                                             > opt_balance_threshold)
        5789                 :                                                 opt_balance_threshold <<= 1;
        5790                 : #endif
        5791               0 :                                         break;
        5792                 :                                 case 'f':
        5793               0 :                                         opt_dirty_max >>= 1;
        5794               0 :                                         break;
        5795                 :                                 case 'F':
        5796               0 :                                         if (opt_dirty_max == 0)
        5797               0 :                                                 opt_dirty_max = 1;
        5798               0 :                                         else if ((opt_dirty_max << 1) != 0)
        5799               0 :                                                 opt_dirty_max <<= 1;
        5800               0 :                                         break;
        5801                 : #ifdef MALLOC_FILL
        5802                 :                                 case 'j':
        5803               0 :                                         opt_junk = false;
        5804               0 :                                         break;
        5805                 :                                 case 'J':
        5806               0 :                                         opt_junk = true;
        5807               0 :                                         break;
        5808                 : #endif
        5809                 : #ifndef MALLOC_STATIC_SIZES
        5810                 :                                 case 'k':
        5811                 :                                         /*
        5812                 :                                          * Chunks always require at least one
        5813                 :                                          * header page, so chunks can never be
        5814                 :                                          * smaller than two pages.
        5815                 :                                          */
        5816                 :                                         if (opt_chunk_2pow > pagesize_2pow + 1)
        5817                 :                                                 opt_chunk_2pow--;
        5818                 :                                         break;
        5819                 :                                 case 'K':
        5820                 :                                         if (opt_chunk_2pow + 1 <
        5821                 :                                             (sizeof(size_t) << 3))
        5822                 :                                                 opt_chunk_2pow++;
        5823                 :                                         break;
        5824                 : #endif
        5825                 :                                 case 'n':
        5826               0 :                                         opt_narenas_lshift--;
        5827               0 :                                         break;
        5828                 :                                 case 'N':
        5829               0 :                                         opt_narenas_lshift++;
        5830               0 :                                         break;
        5831                 : #ifdef MALLOC_PAGEFILE
        5832                 :                                 case 'o':
        5833                 :                                         /* Do not over-commit. */
        5834                 :                                         opt_pagefile = true;
        5835                 :                                         break;
        5836                 :                                 case 'O':
        5837                 :                                         /* Allow over-commit. */
        5838                 :                                         opt_pagefile = false;
        5839                 :                                         break;
        5840                 : #endif
        5841                 :                                 case 'p':
        5842               0 :                                         opt_print_stats = false;
        5843               0 :                                         break;
        5844                 :                                 case 'P':
        5845               0 :                                         opt_print_stats = true;
        5846               0 :                                         break;
        5847                 : #ifndef MALLOC_STATIC_SIZES
        5848                 :                                 case 'q':
        5849                 :                                         if (opt_quantum_2pow > QUANTUM_2POW_MIN)
        5850                 :                                                 opt_quantum_2pow--;
        5851                 :                                         break;
        5852                 :                                 case 'Q':
        5853                 :                                         if (opt_quantum_2pow < pagesize_2pow -
        5854                 :                                             1)
        5855                 :                                                 opt_quantum_2pow++;
        5856                 :                                         break;
        5857                 :                                 case 's':
        5858                 :                                         if (opt_small_max_2pow >
        5859                 :                                             QUANTUM_2POW_MIN)
        5860                 :                                                 opt_small_max_2pow--;
        5861                 :                                         break;
        5862                 :                                 case 'S':
        5863                 :                                         if (opt_small_max_2pow < pagesize_2pow
        5864                 :                                             - 1)
        5865                 :                                                 opt_small_max_2pow++;
        5866                 :                                         break;
        5867                 : #endif
        5868                 : #ifdef MALLOC_UTRACE
        5869                 :                                 case 'u':
        5870               0 :                                         opt_utrace = false;
        5871               0 :                                         break;
        5872                 :                                 case 'U':
        5873               0 :                                         opt_utrace = true;
        5874               0 :                                         break;
        5875                 : #endif
        5876                 : #ifdef MALLOC_SYSV
        5877                 :                                 case 'v':
        5878               0 :                                         opt_sysv = false;
        5879               0 :                                         break;
        5880                 :                                 case 'V':
        5881               0 :                                         opt_sysv = true;
        5882               0 :                                         break;
        5883                 : #endif
        5884                 : #ifdef MALLOC_XMALLOC
        5885                 :                                 case 'x':
        5886               0 :                                         opt_xmalloc = false;
        5887               0 :                                         break;
        5888                 :                                 case 'X':
        5889               0 :                                         opt_xmalloc = true;
        5890               0 :                                         break;
        5891                 : #endif
        5892                 : #ifdef MALLOC_FILL
        5893                 :                                 case 'z':
        5894               0 :                                         opt_zero = false;
        5895               0 :                                         break;
        5896                 :                                 case 'Z':
        5897               0 :                                         opt_zero = true;
        5898               0 :                                         break;
        5899                 : #endif
        5900                 :                                 default: {
        5901                 :                                         char cbuf[2];
        5902                 :
        5903               0 :                                         cbuf[0] = opts[j];
        5904               0 :                                         cbuf[1] = '\0';
        5905               0 :                                         _malloc_message(_getprogname(),
        5906                 :                                             ": (malloc) Unsupported character "
        5907                 :                                             "in malloc options: '", cbuf,
        5908                 :                                             "'\n");
        5909                 :                                 }
        5910                 :                                 }
        5911                 :                         }
        5912                 :                 }
        5913                 :         }
        5914                 :
        5915                 :         /* Take care to call atexit() only once. */
        5916           18669 :         if (opt_print_stats) {
        5917                 : #ifndef MOZ_MEMORY_WINDOWS
        5918                 :                 /* Print statistics at exit. */
        5919               0 :                 atexit(malloc_print_stats);
        5920                 : #endif
        5921                 :         }
        5922                 :
        5923                 : #if !defined(MOZ_MEMORY_WINDOWS) && !defined(MOZ_MEMORY_DARWIN)
        5924                 :         /* Prevent potential deadlock on malloc locks after fork. */
        5925           18669 :         pthread_atfork(_malloc_prefork, _malloc_postfork, _malloc_postfork);
        5926                 : #endif
        5927                 :
        5928                 : #ifndef MALLOC_STATIC_SIZES
        5929                 :         /* Set variables according to the value of opt_small_max_2pow. */
        5930                 :         if (opt_small_max_2pow < opt_quantum_2pow)
        5931                 :                 opt_small_max_2pow = opt_quantum_2pow;
        5932                 :         small_max = (1U << opt_small_max_2pow);
        5933                 :
        5934                 :         /* Set bin-related variables. */
        5935                 :         bin_maxclass = (pagesize >> 1);
        5936                 :         assert(opt_quantum_2pow >= TINY_MIN_2POW);
        5937                 :         ntbins = opt_quantum_2pow - TINY_MIN_2POW;
        5938                 :         assert(ntbins <= opt_quantum_2pow);
        5939                 :         nqbins = (small_max >> opt_quantum_2pow);
        5940                 :         nsbins = pagesize_2pow - opt_small_max_2pow - 1;
        5941                 :
        5942                 :         /* Set variables according to the value of opt_quantum_2pow. */
        5943                 :         quantum = (1U << opt_quantum_2pow);
        5944                 :         quantum_mask = quantum - 1;
        5945                 :         if (ntbins > 0)
        5946                 :                 small_min = (quantum >> 1) + 1;
        5947                 :         else
        5948                 :                 small_min = 1;
        5949                 :         assert(small_min <= quantum);
        5950                 :
        5951                 :         /* Set variables according to the value of opt_chunk_2pow. */
        5952                 :         chunksize = (1LU << opt_chunk_2pow);
        5953                 :         chunksize_mask = chunksize - 1;
        5954                 :         chunk_npages = (chunksize >> pagesize_2pow);
        5955                 :
        5956                 :         arena_chunk_header_npages = calculate_arena_header_pages();
        5957                 :         arena_maxclass = calculate_arena_maxclass();
        5958                 : #endif
        5959                 :
        5960                 : #ifdef JEMALLOC_USES_MAP_ALIGN
        5961                 :         /*
        5962                 :          * When using MAP_ALIGN, the alignment parameter must be a power of two
        5963                 :          * multiple of the system pagesize, or mmap will fail.
        5964                 :          */
        5965                 :         assert((chunksize % pagesize) == 0);
        5966                 :         assert((1 << (ffs(chunksize / pagesize) - 1)) == (chunksize/pagesize));
        5967                 : #endif
        5968                 :
        5969           18669 :         UTRACE(0, 0, 0);
        5970                 :
        5971                 : #ifdef MALLOC_STATS
        5972           18669 :         memset(&stats_chunks, 0, sizeof(chunk_stats_t));
        5973                 : #endif
        5974                 :
        5975                 :         /* Various sanity checks that regard configuration. */
        5976           18669 :         assert(quantum >= sizeof(void *));
        5977           18669 :         assert(quantum <= pagesize);
        5978           18669 :         assert(chunksize >= pagesize);
        5979           18669 :         assert(quantum * 4 <= chunksize);
        5980                 :
        5981                 :         /* Initialize chunks data. */
        5982           18669 :         malloc_mutex_init(&huge_mtx);
        5983           18669 :         extent_tree_ad_new(&huge);
        5984                 : #ifdef MALLOC_STATS
        5985           18669 :         huge_nmalloc = 0;
        5986           18669 :         huge_ndalloc = 0;
        5987           18669 :         huge_allocated = 0;
        5988                 : #endif
        5989                 :
        5990                 :         /* Initialize base allocation data structures. */
        5991                 : #ifdef MALLOC_STATS
        5992           18669 :         base_mapped = 0;
        5993           18669 :         base_committed = 0;
        5994                 : #endif
        5995           18669 :         base_nodes = NULL;
        5996           18669 :         malloc_mutex_init(&base_mtx);
        5997                 :
        5998                 : #ifdef MOZ_MEMORY_NARENAS_DEFAULT_ONE
        5999           18669 :         narenas = 1;
        6000                 : #else
        6001                 :         if (ncpus > 1) {
        6002                 :                 /*
        6003                 :                  * For SMP systems, create four times as many arenas as there
        6004                 :                  * are CPUs by default.
        6005                 :                  */
        6006                 :                 opt_narenas_lshift += 2;
        6007                 :         }
        6008                 :
        6009                 :         /* Determine how many arenas to use. */
        6010                 :         narenas = ncpus;
        6011                 : #endif
        6012           18669 :         if (opt_narenas_lshift > 0) {
        6013               0 :                 if ((narenas << opt_narenas_lshift) > narenas)
        6014               0 :                         narenas <<= opt_narenas_lshift;
        6015                 :                 /*
        6016                 :                  * Make sure not to exceed the limits of what base_alloc() can
        6017                 :                  * handle.
        6018                 :                  */
        6019               0 :                 if (narenas * sizeof(arena_t *) > chunksize)
        6020               0 :                         narenas = chunksize / sizeof(arena_t *);
        6021           18669 :         } else if (opt_narenas_lshift < 0) {
        6022               0 :                 if ((narenas >> -opt_narenas_lshift) < narenas)
        6023               0 :                         narenas >>= -opt_narenas_lshift;
        6024                 :                 /* Make sure there is at least one arena. */
        6025               0 :                 if (narenas == 0)
        6026               0 :                         narenas = 1;
        6027                 :         }
        6028                 : #ifdef MALLOC_BALANCE
        6029                 :         assert(narenas != 0);
        6030                 :         for (narenas_2pow = 0;
        6031                 :              (narenas >> (narenas_2pow + 1)) != 0;
        6032                 :              narenas_2pow++);
        6033                 : #endif
        6034                 :
        6035                 : #ifdef NO_TLS
        6036                 :         if (narenas > 1) {
        6037                 :                 static const unsigned primes[] = {1, 3, 5, 7, 11, 13, 17, 19,
        6038                 :                     23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83,
        6039                 :                     89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149,
        6040                 :                     151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211,
        6041                 :                     223, 227, 229, 233, 239, 241, 251, 257, 263};
        6042                 :                 unsigned nprimes, parenas;
        6043                 :
        6044                 :                 /*
        6045                 :                  * Pick a prime number of hash arenas that is more than narenas
        6046                 :                  * so that direct hashing of pthread_self() pointers tends to
        6047                 :                  * spread allocations evenly among the arenas.
        6048                 :                  */
        6049                 :                 assert((narenas & 1) == 0); /* narenas must be even. */
        6050                 :                 nprimes = (sizeof(primes) >> SIZEOF_INT_2POW);
        6051                 :                 parenas = primes[nprimes - 1]; /* In case not enough primes. */
        6052                 :                 for (i = 1; i < nprimes; i++) {
        6053                 :                         if (primes[i] > narenas) {
        6054                 :                                 parenas = primes[i];
        6055                 :                                 break;
        6056                 :                         }
        6057                 :                 }
        6058                 :                 narenas = parenas;
        6059                 :         }
        6060                 : #endif
        6061                 :
        6062                 : #ifndef NO_TLS
        6063                 : #  ifndef MALLOC_BALANCE
        6064           18669 :         next_arena = 0;
        6065                 : #  endif
        6066                 : #endif
        6067                 :
        6068                 :         /* Allocate and initialize arenas. */
        6069           18669 :         arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas);
        6070           18669 :         if (arenas == NULL) {
        6071                 : #ifndef MOZ_MEMORY_WINDOWS
        6072               0 :                 malloc_mutex_unlock(&init_lock);
        6073                 : #endif
        6074               0 :                 return (true);
        6075                 :         }
        6076                 :         /*
        6077                 :          * Zero the array.  In practice, this should always be pre-zeroed,
        6078                 :          * since it was just mmap()ed, but let's be sure.
        6079                 :          */
        6080           18669 :         memset(arenas, 0, sizeof(arena_t *) * narenas);
        6081                 :
        6082                 :         /*
        6083                 :          * Initialize one arena here.  The rest are lazily created in
        6084                 :          * choose_arena_hard().
        6085                 :          */
        6086           18669 :         arenas_extend(0);
        6087           18669 :         if (arenas[0] == NULL) {
        6088                 : #ifndef MOZ_MEMORY_WINDOWS
        6089               0 :                 malloc_mutex_unlock(&init_lock);
        6090                 : #endif
        6091               0 :                 return (true);
        6092                 :         }
        6093                 : #ifndef NO_TLS
        6094                 :         /*
        6095                 :          * Assign the initial arena to the initial thread, in order to avoid
        6096                 :          * spurious creation of an extra arena if the application switches to
        6097                 :          * threaded mode.
        6098                 :          */
        6099                 : #ifdef MOZ_MEMORY_WINDOWS
        6100                 :         TlsSetValue(tlsIndex, arenas[0]);
        6101                 : #else
        6102           18669 :         arenas_map = arenas[0];
        6103                 : #endif
        6104                 : #endif
        6105                 :
        6106                 :         /*
        6107                 :          * Seed here for the initial thread, since choose_arena_hard() is only
        6108                 :          * called for other threads.  The seed value doesn't really matter.
        6109                 :          */
        6110                 : #ifdef MALLOC_BALANCE
        6111                 :         SPRN(balance, 42);
        6112                 : #endif
        6113                 :
        6114           18669 :         malloc_spin_init(&arenas_lock);
        6115                 :
        6116                 : #ifdef MALLOC_VALIDATE
        6117           18669 :         chunk_rtree = malloc_rtree_new((SIZEOF_PTR << 3) - opt_chunk_2pow);
        6118           18669 :         if (chunk_rtree == NULL)
        6119               0 :                 return (true);
        6120                 : #endif
        6121                 :
        6122           18669 :         malloc_initialized = true;
        6123                 :
        6124                 : #if defined(NEEDS_PTHREAD_MMAP_UNALIGNED_TSD)
        6125                 :         if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) {
        6126                 :                 malloc_printf("<jemalloc>: Error in pthread_key_create()\n");
        6127                 :         }
        6128                 : #endif
        6129                 :
        6130                 : #ifdef MOZ_MEMORY_DARWIN
        6131                 :         /*
        6132                 :         * Overwrite the default memory allocator to use jemalloc everywhere.
        6133                 :         */
        6134                 :         default_zone = malloc_default_zone();
        6135                 :
        6136                 :         /*
        6137                 :          * We only use jemalloc with MacOS 10.6 and 10.7.  jemalloc is disabled
        6138                 :          * on 32-bit builds (10.5 and 32-bit 10.6) due to bug 702250, an
        6139                 :          * apparent MacOS bug.  In fact, this code isn't even compiled on
        6140                 :          * 32-bit builds.
        6141                 :          *
        6142                 :          * We'll have to update our code to work with newer versions, because
        6143                 :          * the malloc zone layout is likely to change.
        6144                 :          */
        6145                 :
        6146                 :         osx_use_jemalloc = (default_zone->version == SNOW_LEOPARD_MALLOC_ZONE_T_VERSION ||
        6147                 :                             default_zone->version == LION_MALLOC_ZONE_T_VERSION);
        6148                 :
        6149                 :         /* Allow us dynamically turn off jemalloc for testing. */
        6150                 :         if (getenv("NO_MAC_JEMALLOC")) {
        6151                 :                 osx_use_jemalloc = false;
        6152                 : #ifdef __i386__
        6153                 :                 malloc_printf("Warning: NO_MAC_JEMALLOC has no effect on "
        6154                 :                               "i386 machines (such as this one).\n");
        6155                 : #endif
        6156                 :         }
        6157                 :
        6158                 :         if (osx_use_jemalloc) {
        6159                 :                 /*
        6160                 :                  * Convert the default szone to an "overlay zone" that is capable
        6161                 :                  * of deallocating szone-allocated objects, but allocating new
        6162                 :                  * objects from jemalloc.
        6163                 :                  */
        6164                 :                 size_t size = zone_version_size(default_zone->version);
        6165                 :                 szone2ozone(default_zone, size);
        6166                 :         }
        6167                 :         else {
        6168                 :                 szone = default_zone;
        6169                 :         }
        6170                 : #endif
        6171                 :
        6172                 : #ifndef MOZ_MEMORY_WINDOWS
        6173           18669 :         malloc_mutex_unlock(&init_lock);
        6174                 : #endif
        6175           18669 :         return (false);
        6176                 : }
        6177                 :
        6178                 : /* XXX Why not just expose malloc_print_stats()? */
        6179                 : #ifdef MOZ_MEMORY_WINDOWS
        6180                 : void
        6181                 : malloc_shutdown()
        6182                 : {
        6183                 :
        6184                 :         malloc_print_stats();
        6185                 : }
        6186                 : #endif
        6187                 :
        6188                 : /*
        6189                 :  * End general internal functions.
        6190                 :  */
        6191                 : /******************************************************************************/
        6192                 : /*
        6193                 :  * Begin malloc(3)-compatible functions.
        6194                 :  */
        6195                 :
        6196                 : /*
        6197                 :  * Mangle standard interfaces, in order to avoid linking problems.
        6198                 :  */
        6199                 : #if defined(MOZ_MEMORY_DARWIN) || defined(MOZ_MEMORY_WINDOWS) || \
        6200                 :     defined(MOZ_MEMORY_ANDROID)
        6201                 :
        6202                 : #ifdef MOZ_MEMORY_ANDROID
        6203                 : /*
        6204                 :  * On Android, we use __wrap_* instead of je_* to accomodate with the
        6205                 :  * linker's --wrap option we use. That option prefixes the function
        6206                 :  * names it is given with __wrap_.
        6207                 :  */
        6208                 : #define wrap(a) __wrap_ ## a
        6209                 : #else
        6210                 : #define wrap(a) je_ ## a
        6211                 : #endif
        6212                 :
        6213                 : #define malloc(a)               wrap(malloc)(a)
        6214                 : #define memalign(a, b)          wrap(memalign)(a, b)
        6215                 : #define posix_memalign(a, b, c) wrap(posix_memalign)(a, b, c)
        6216                 : #define valloc(a)               wrap(valloc)(a)
        6217                 : #define calloc(a, b)            wrap(calloc)(a, b)
        6218                 : #define realloc(a, b)           wrap(realloc)(a, b)
        6219                 : #define free(a)                 wrap(free)(a)
        6220                 : #define malloc_usable_size(a)   wrap(malloc_usable_size)(a)
        6221                 : #endif
        6222                 :
        6223                 : /*
        6224                 :  * Even though we compile with MOZ_MEMORY, we may have to dynamically decide
        6225                 :  * not to use jemalloc, as discussed above. However, we call jemalloc
        6226                 :  * functions directly from mozalloc. Since it's pretty dangerous to mix the
        6227                 :  * allocators, we need to call the OSX allocators from the functions below,
        6228                 :  * when osx_use_jemalloc is not (dynamically) set.
        6229                 :  *
        6230                 :  * Note that we assume jemalloc is enabled on i386.  This is safe because the
        6231                 :  * only i386 versions of MacOS are 10.5 and 10.6, which we support.  We have to
        6232                 :  * do this because madvise isn't in the malloc zone struct for 10.5.
        6233                 :  *
        6234                 :  * This means that NO_MAC_JEMALLOC doesn't work on i386.
        6235                 :  */
        6236                 : #if defined(MOZ_MEMORY_DARWIN) && !defined(__i386__)
        6237                 : #define DARWIN_ONLY(A) if (!osx_use_jemalloc) { A; }
        6238                 : #else
        6239                 : #define DARWIN_ONLY(A)
        6240                 : #endif
        6241                 :
        6242                 : void *
        6243                 : malloc(size_t size)
        6244                 : {
        6245                 :         void *ret;
        6246                 :
        6247                 :         DARWIN_ONLY(return (szone->malloc)(szone, size));
        6248                 :
        6249        37094556 :         if (malloc_init()) {
        6250               0 :                 ret = NULL;
        6251               0 :                 goto RETURN;
        6252                 :         }
        6253                 :
        6254        37094556 :         if (size == 0) {
        6255                 : #ifdef MALLOC_SYSV
        6256              22 :                 if (opt_sysv == false)
        6257                 : #endif
        6258              22 :                         size = 1;
        6259                 : #ifdef MALLOC_SYSV
        6260                 :                 else {
        6261               0 :                         ret = NULL;
        6262               0 :                         goto RETURN;
        6263                 :                 }
        6264                 : #endif
        6265                 :         }
        6266                 :
        6267        37094556 :         ret = imalloc(size);
        6268                 :
        6269                 : RETURN:
        6270        37094556 :         if (ret == NULL) {
        6271                 : #ifdef MALLOC_XMALLOC
        6272               0 :                 if (opt_xmalloc) {
        6273               0 :                         _malloc_message(_getprogname(),
        6274                 :                             ": (malloc) Error in malloc(): out of memory\n", "",
        6275                 :                             "");
        6276               0 :                         abort();
        6277                 :                 }
        6278                 : #endif
        6279               0 :                 errno = ENOMEM;
        6280                 :         }
        6281                 :
        6282        37094556 :         UTRACE(0, size, ret);
        6283        37094556 :         return (ret);
        6284                 : }
        6285                 :
        6286                 : /*
        6287                 :  * In ELF systems the default visibility allows symbols to be preempted at
        6288                 :  * runtime. This in turn prevents the uses of memalign in this file from being
        6289                 :  * optimized. What we do in here is define two aliasing symbols (they point to
        6290                 :  * the same code): memalign and memalign_internal. The internal version has
        6291                 :  * hidden visibility and is used in every reference from this file.
        6292                 :  *
        6293                 :  * For more information on this technique, see section 2.2.7 (Avoid Using
        6294                 :  * Exported Symbols) in http://www.akkadia.org/drepper/dsohowto.pdf.
        6295                 :  */
        6296                 :
        6297                 : #if defined(__GNUC__) && !defined(MOZ_MEMORY_DARWIN)
        6298                 : #define MOZ_MEMORY_ELF
        6299                 : #endif
        6300                 :
        6301                 : #ifdef MOZ_MEMORY_SOLARIS
        6302                 : #  ifdef __SUNPRO_C
        6303                 : void *
        6304                 : memalign(size_t alignment, size_t size);
        6305                 : #pragma no_inline(memalign)
        6306                 : #  elif (defined(__GNUC__))
        6307                 : __attribute__((noinline))
        6308                 : #  endif
        6309                 : #else
        6310                 : #if (defined(MOZ_MEMORY_ELF))
        6311                 : __attribute__((visibility ("hidden")))
        6312                 : #endif
        6313                 : #endif
        6314                 :
        6315                 :
        6316                 : #ifdef MOZ_MEMORY_ELF
        6317                 : #define MEMALIGN memalign_internal
        6318                 : #else
        6319                 : #define MEMALIGN memalign
        6320                 : #endif
        6321                 :
        6322                 : void *
        6323               0 : MEMALIGN(size_t alignment, size_t size)
        6324                 : {
        6325                 :         void *ret;
        6326                 :
        6327                 :         DARWIN_ONLY(return (szone->memalign)(szone, alignment, size));
        6328                 :
        6329               0 :         assert(((alignment - 1) & alignment) == 0);
        6330                 :
        6331               0 :         if (malloc_init()) {
        6332               0 :                 ret = NULL;
        6333               0 :                 goto RETURN;
        6334                 :         }
        6335                 :
        6336               0 :         if (size == 0) {
        6337                 : #ifdef MALLOC_SYSV
        6338               0 :                 if (opt_sysv == false)
        6339                 : #endif
        6340               0 :                         size = 1;
        6341                 : #ifdef MALLOC_SYSV
        6342                 :                 else {
        6343               0 :                         ret = NULL;
        6344               0 :                         goto RETURN;
        6345                 :                 }
        6346                 : #endif
        6347                 :         }
        6348                 :
        6349               0 :         alignment = alignment < sizeof(void*) ? sizeof(void*) : alignment;
        6350               0 :         ret = ipalloc(alignment, size);
        6351                 :
        6352                 : RETURN:
        6353                 : #ifdef MALLOC_XMALLOC
        6354               0 :         if (opt_xmalloc && ret == NULL) {
        6355               0 :                 _malloc_message(_getprogname(),
        6356                 :                 ": (malloc) Error in memalign(): out of memory\n", "", "");
        6357               0 :                 abort();
        6358                 :         }
        6359                 : #endif
        6360               0 :         UTRACE(0, size, ret);
        6361               0 :         return (ret);
        6362                 : }
        6363                 :
        6364                 : #ifdef MOZ_MEMORY_ELF
        6365                 : extern void *
        6366                 : memalign(size_t alignment, size_t size) __attribute__((alias ("memalign_internal"), visibility ("default")));
        6367                 : #endif
        6368                 :
        6369                 : int
        6370                 : posix_memalign(void **memptr, size_t alignment, size_t size)
        6371                 : {
        6372                 :         void *result;
        6373                 :
        6374                 :         /* Make sure that alignment is a large enough power of 2. */
        6375               0 :         if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void *)) {
        6376                 : #ifdef MALLOC_XMALLOC
        6377               0 :                 if (opt_xmalloc) {
        6378               0 :                         _malloc_message(_getprogname(),
        6379                 :                             ": (malloc) Error in posix_memalign(): "
        6380                 :                             "invalid alignment\n", "", "");
        6381               0 :                         abort();
        6382                 :                 }
        6383                 : #endif
        6384               0 :                 return (EINVAL);
        6385                 :         }
        6386                 :
        6387                 :         /* The 0-->1 size promotion is done in the memalign() call below */
        6388                 :
        6389               0 :         result = MEMALIGN(alignment, size);
        6390                 :
        6391               0 :         if (result == NULL)
        6392               0 :                 return (ENOMEM);
        6393                 :
        6394               0 :         *memptr = result;
        6395               0 :         return (0);
        6396                 : }
        6397                 :
        6398                 : void *
        6399                 : valloc(size_t size)
        6400                 : {
        6401               0 :         return (MEMALIGN(pagesize, size));
        6402                 : }
        6403                 :
        6404                 : void *
        6405                 : calloc(size_t num, size_t size)
        6406                 : {
        6407                 :         void *ret;
        6408                 :         size_t num_size;
        6409                 :
        6410                 :         DARWIN_ONLY(return (szone->calloc)(szone, num, size));
        6411                 :
        6412         2256419 :         if (malloc_init()) {
        6413               0 :                 num_size = 0;
        6414               0 :                 ret = NULL;
        6415               0 :                 goto RETURN;
        6416                 :         }
        6417                 :
        6418         2256419 :         num_size = num * size;
        6419         2256419 :         if (num_size == 0) {
        6420                 : #ifdef MALLOC_SYSV
        6421               0 :                 if ((opt_sysv == false) && ((num == 0) || (size == 0)))
        6422                 : #endif
        6423               0 :                         num_size = 1;
        6424                 : #ifdef MALLOC_SYSV
        6425                 :                 else {
        6426               0 :                         ret = NULL;
        6427               0 :                         goto RETURN;
        6428                 :                 }
        6429                 : #endif
        6430                 :         /*
        6431                 :          * Try to avoid division here.  We know that it isn't possible to
        6432                 :          * overflow during multiplication if neither operand uses any of the
        6433                 :          * most significant half of the bits in a size_t.
        6434                 :          */
        6435         2256419 :         } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2)))
        6436             551 :             && (num_size / size != num)) {
        6437                 :                 /* size_t overflow. */
        6438               0 :                 ret = NULL;
        6439               0 :                 goto RETURN;
        6440                 :         }
        6441                 :
        6442         2256419 :         ret = icalloc(num_size);
        6443                 :
        6444                 : RETURN:
        6445         2256419 :         if (ret == NULL) {
        6446                 : #ifdef MALLOC_XMALLOC
        6447              18 :                 if (opt_xmalloc) {
        6448               0 :                         _malloc_message(_getprogname(),
        6449                 :                             ": (malloc) Error in calloc(): out of memory\n", "",
        6450                 :                             "");
        6451               0 :                         abort();
        6452                 :                 }
        6453                 : #endif
        6454              18 :                 errno = ENOMEM;
        6455                 :         }
        6456                 :
        6457         2256419 :         UTRACE(0, num_size, ret);
        6458         2256419 :         return (ret);
        6459                 : }
        6460                 :
        6461                 : void *
        6462                 : realloc(void *ptr, size_t size)
        6463                 : {
        6464                 :         void *ret;
        6465                 :
        6466                 :         DARWIN_ONLY(return (szone->realloc)(szone, ptr, size));
        6467                 :
        6468         7065387 :         if (size == 0) {
        6469                 : #ifdef MALLOC_SYSV
        6470               0 :                 if (opt_sysv == false)
        6471                 : #endif
        6472               0 :                         size = 1;
        6473                 : #ifdef MALLOC_SYSV
        6474                 :                 else {
        6475               0 :                         if (ptr != NULL)
        6476               0 :                                 idalloc(ptr);
        6477               0 :                         ret = NULL;
        6478               0 :                         goto RETURN;
        6479                 :                 }
        6480                 : #endif
        6481                 :         }
        6482                 :
        6483         7065387 :         if (ptr != NULL) {
        6484         2301663 :                 assert(malloc_initialized);
        6485                 :
        6486         2301663 :                 ret = iralloc(ptr, size);
        6487                 :
        6488         2301663 :                 if (ret == NULL) {
        6489                 : #ifdef MALLOC_XMALLOC
        6490               0 :                         if (opt_xmalloc) {
        6491               0 :                                 _malloc_message(_getprogname(),
        6492                 :                                     ": (malloc) Error in realloc(): out of "
        6493                 :                                     "memory\n", "", "");
        6494               0 :                                 abort();
        6495                 :                         }
        6496                 : #endif
        6497               0 :                         errno = ENOMEM;
        6498                 :                 }
        6499                 :         } else {
        6500         4763724 :                 if (malloc_init())
        6501               0 :                         ret = NULL;
        6502                 :                 else
        6503         4763724 :                         ret = imalloc(size);
        6504                 :
        6505         4763724 :                 if (ret == NULL) {
        6506                 : #ifdef MALLOC_XMALLOC
        6507               0 :                         if (opt_xmalloc) {
        6508               0 :                                 _malloc_message(_getprogname(),
        6509                 :                                     ": (malloc) Error in realloc(): out of "
        6510                 :                                     "memory\n", "", "");
        6511               0 :                                 abort();
        6512                 :                         }
        6513                 : #endif
        6514               0 :                         errno = ENOMEM;
        6515                 :                 }
        6516                 :         }
        6517                 :
        6518                 : #ifdef MALLOC_SYSV
        6519                 : RETURN:
        6520                 : #endif
        6521         7065387 :         UTRACE(ptr, size, ret);
        6522         7065387 :         return (ret);
        6523                 : }
        6524                 :
        6525                 : void
        6526                 : free(void *ptr)
        6527                 : {
        6528                 :         size_t offset;
        6529                 :
        6530                 :         DARWIN_ONLY((szone->free)(szone, ptr); return);
        6531                 :
        6532        50933286 :         UTRACE(ptr, 0, 0);
        6533                 :
        6534                 :         /*
        6535                 :          * A version of idalloc that checks for NULL pointer but only for
        6536                 :          * huge allocations assuming that CHUNK_ADDR2OFFSET(NULL) == 0.
        6537                 :          */
        6538                 :         assert(CHUNK_ADDR2OFFSET(NULL) == 0);
        6539        50933289 :         offset = CHUNK_ADDR2OFFSET(ptr);
        6540        50933289 :         if (offset != 0)
        6541        42036375 :                 arena_dalloc(ptr, offset);
        6542         8896914 :         else if (ptr != NULL)
        6543            2916 :                 huge_dalloc(ptr);
        6544        50933300 : }
        6545                 :
        6546                 : /*
        6547                 :  * End malloc(3)-compatible functions.
        6548                 :  */
        6549                 : /******************************************************************************/
        6550                 : /*
        6551                 :  * Begin non-standard functions.
        6552                 :  */
        6553                 :
        6554                 : /* This was added by Mozilla for use by SQLite. */
        6555                 : #ifdef MOZ_MEMORY_DARWIN
        6556                 : static
        6557                 : #endif
        6558                 : size_t
        6559               0 : je_malloc_good_size(size_t size)
        6560                 : {
        6561                 :         /*
        6562                 :          * This duplicates the logic in imalloc(), arena_malloc() and
        6563                 :          * arena_malloc_small().
        6564                 :          */
        6565               0 :         if (size < small_min) {
        6566                 :                 /* Small (tiny). */
        6567               0 :                 size = pow2_ceil(size);
        6568                 :                 /*
        6569                 :                  * We omit the #ifdefs from arena_malloc_small() --
        6570                 :                  * it can be inaccurate with its size in some cases, but this
        6571                 :                  * function must be accurate.
        6572                 :                  */
        6573               0 :                 if (size < (1U << TINY_MIN_2POW))
        6574               0 :                         size = (1U << TINY_MIN_2POW);
        6575               0 :         } else if (size <= small_max) {
        6576                 :                 /* Small (quantum-spaced). */
        6577               0 :                 size = QUANTUM_CEILING(size);
        6578               0 :         } else if (size <= bin_maxclass) {
        6579                 :                 /* Small (sub-page). */
        6580               0 :                 size = pow2_ceil(size);
        6581               0 :         } else if (size <= arena_maxclass) {
        6582                 :                 /* Large. */
        6583               0 :                 size = PAGE_CEILING(size);
        6584                 :         } else {
        6585                 :                 /*
        6586                 :                  * Huge.  We use PAGE_CEILING to get psize, instead of using
        6587                 :                  * CHUNK_CEILING to get csize.  This ensures that this
        6588                 :                  * malloc_usable_size(malloc(n)) always matches
        6589                 :                  * je_malloc_good_size(n).
        6590                 :                  */
        6591               0 :                 size = PAGE_CEILING(size);
        6592                 :         }
        6593               0 :         return size;
        6594                 : }
        6595                 :
        6596                 :
        6597                 : #ifdef MOZ_MEMORY_ANDROID
        6598                 : size_t
        6599                 : malloc_usable_size(void *ptr)
        6600                 : #else
        6601                 : size_t
        6602               0 : malloc_usable_size(const void *ptr)
        6603                 : #endif
        6604                 : {
        6605                 :         DARWIN_ONLY(return (szone->size)(szone, ptr));
        6606                 :
        6607                 : #ifdef MALLOC_VALIDATE
        6608               0 :         return (isalloc_validate(ptr));
        6609                 : #else
        6610                 :         assert(ptr != NULL);
        6611                 :
        6612                 :         return (isalloc(ptr));
        6613                 : #endif
        6614                 : }
        6615                 :
        6616                 : void
        6617               0 : jemalloc_stats(jemalloc_stats_t *stats)
        6618                 : {
        6619                 :         size_t i;
        6620                 :
        6621               0 :         assert(stats != NULL);
        6622                 :
        6623                 :         /*
        6624                 :          * Gather runtime settings.
        6625                 :          */
        6626               0 :         stats->opt_abort = opt_abort;
        6627               0 :         stats->opt_junk =
        6628                 : #ifdef MALLOC_FILL
        6629                 :             opt_junk ? true :
        6630                 : #endif
        6631                 :             false;
        6632               0 :         stats->opt_utrace =
        6633                 : #ifdef MALLOC_UTRACE
        6634                 :             opt_utrace ? true :
        6635                 : #endif
        6636                 :             false;
        6637               0 :         stats->opt_sysv =
        6638                 : #ifdef MALLOC_SYSV
        6639                 :             opt_sysv ? true :
        6640                 : #endif
        6641                 :             false;
        6642               0 :         stats->opt_xmalloc =
        6643                 : #ifdef MALLOC_XMALLOC
        6644                 :             opt_xmalloc ? true :
        6645                 : #endif
        6646                 :             false;
        6647               0 :         stats->opt_zero =
        6648                 : #ifdef MALLOC_FILL
        6649                 :             opt_zero ? true :
        6650                 : #endif
        6651                 :             false;
        6652               0 :         stats->narenas = narenas;
        6653               0 :         stats->balance_threshold =
        6654                 : #ifdef MALLOC_BALANCE
        6655                 :             opt_balance_threshold
        6656                 : #else
        6657                 :             SIZE_T_MAX
        6658                 : #endif
        6659                 :             ;
        6660               0 :         stats->quantum = quantum;
        6661               0 :         stats->small_max = small_max;
        6662               0 :         stats->large_max = arena_maxclass;
        6663               0 :         stats->chunksize = chunksize;
        6664               0 :         stats->dirty_max = opt_dirty_max;
        6665                 :
        6666                 :         /*
        6667                 :          * Gather current memory usage statistics.
        6668                 :          */
        6669               0 :         stats->mapped = 0;
        6670               0 :         stats->committed = 0;
        6671               0 :         stats->allocated = 0;
        6672               0 :         stats->dirty = 0;
        6673                 :
        6674                 :         /* Get huge mapped/allocated. */
        6675               0 :         malloc_mutex_lock(&huge_mtx);
        6676               0 :         stats->mapped += stats_chunks.curchunks * chunksize;
        6677               0 :         stats->committed += huge_allocated;
        6678               0 :         stats->allocated += huge_allocated;
        6679               0 :         malloc_mutex_unlock(&huge_mtx);
        6680                 :
        6681                 :         /* Get base mapped. */
        6682               0 :         malloc_mutex_lock(&base_mtx);
        6683               0 :         stats->mapped += base_mapped;
        6684               0 :         assert(base_committed <= base_mapped);
        6685               0 :         stats->committed += base_committed;
        6686               0 :         malloc_mutex_unlock(&base_mtx);
        6687                 :
        6688                 :         /* Iterate over arenas and their chunks. */
        6689               0 :         for (i = 0; i < narenas; i++) {
        6690               0 :                 arena_t *arena = arenas[i];
        6691               0 :                 if (arena != NULL) {
        6692               0 :                         malloc_spin_lock(&arena->lock);
        6693               0 :                         stats->allocated += arena->stats.allocated_small;
        6694               0 :                         stats->allocated += arena->stats.allocated_large;
        6695               0 :                         stats->committed += (arena->stats.committed <<
        6696                 :                             pagesize_2pow);
        6697               0 :                         stats->dirty += (arena->ndirty << pagesize_2pow);
        6698               0 :                         malloc_spin_unlock(&arena->lock);
        6699                 :                 }
        6700                 :         }
        6701                 :
        6702               0 :         assert(stats->mapped >= stats->committed);
        6703               0 :         assert(stats->committed >= stats->allocated);
        6704               0 : }
        6705                 :
        6706                 : #ifdef MALLOC_DOUBLE_PURGE
        6707                 :
        6708                 : /* Explicitly remove all of this chunk's MADV_FREE'd pages from memory. */
        6709                 : static void
        6710                 : hard_purge_chunk(arena_chunk_t *chunk)
        6711                 : {
        6712                 :         /* See similar logic in arena_purge(). */
        6713                 :
        6714                 :         size_t i;
        6715                 :         for (i = arena_chunk_header_npages; i < chunk_npages; i++) {
        6716                 :                 /* Find all adjacent pages with CHUNK_MAP_MADVISED set. */
        6717                 :                 size_t npages;
        6718                 :                 for (npages = 0;
        6719                 :                      chunk->map[i + npages].bits & CHUNK_MAP_MADVISED && i + npages < chunk_npages;
        6720                 :                      npages++) {
        6721                 :                         /* Turn off the chunk's MADV_FREED bit and turn on its
        6722                 :                          * DECOMMITTED bit. */
        6723                 :                         assert(!(chunk->map[i + npages].bits & CHUNK_MAP_DECOMMITTED));
        6724                 :                         chunk->map[i + npages].bits ^= CHUNK_MAP_MADVISED_OR_DECOMMITTED;
        6725                 :                 }
        6726                 :
        6727                 :                 /* We could use mincore to find out which pages are actually
        6728                 :                  * present, but it's not clear that's better. */
        6729                 :                 if (npages > 0) {
        6730                 :                         pages_decommit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
        6731                 :                         pages_commit(((char*)chunk) + (i << pagesize_2pow), npages << pagesize_2pow);
        6732                 :                 }
        6733                 :                 i += npages;
        6734                 :         }
        6735                 : }
        6736                 :
        6737                 : /* Explicitly remove all of this arena's MADV_FREE'd pages from memory. */
        6738                 : static void
        6739                 : hard_purge_arena(arena_t *arena)
        6740                 : {
        6741                 :         malloc_spin_lock(&arena->lock);
        6742                 :
        6743                 :         while (!LinkedList_IsEmpty(&arena->chunks_madvised)) {
        6744                 :                 LinkedList* next = arena->chunks_madvised.next;
        6745                 :                 arena_chunk_t *chunk =
        6746                 :                         LinkedList_Get(arena->chunks_madvised.next,
        6747                 :                                        arena_chunk_t, chunks_madvised_elem);
        6748                 :                 hard_purge_chunk(chunk);
        6749                 :                 LinkedList_Remove(&chunk->chunks_madvised_elem);
        6750                 :         }
        6751                 :
        6752                 :         malloc_spin_unlock(&arena->lock);
        6753                 : }
        6754                 :
        6755                 : void
        6756                 : jemalloc_purge_freed_pages()
        6757                 : {
        6758                 :         size_t i;
        6759                 :         for (i = 0; i < narenas; i++) {
        6760                 :                 arena_t *arena = arenas[i];
        6761                 :                 if (arena != NULL)
        6762                 :                         hard_purge_arena(arena);
        6763                 :         }
        6764                 : }
        6765                 :
        6766                 : #else /* !defined MALLOC_DOUBLE_PURGE */
        6767                 :
        6768                 : void
        6769               0 : jemalloc_purge_freed_pages()
        6770                 : {
        6771                 :         /* Do nothing. */
        6772               0 : }
        6773                 :
        6774                 : #endif /* defined MALLOC_DOUBLE_PURGE */
        6775                 :
        6776                 :
        6777                 :
        6778                 : #ifdef MOZ_MEMORY_WINDOWS
        6779                 : void*
        6780                 : _recalloc(void *ptr, size_t count, size_t size)
        6781                 : {
        6782                 :         size_t oldsize = (ptr != NULL) ? isalloc(ptr) : 0;
        6783                 :         size_t newsize = count * size;
        6784                 :
        6785                 :         /*
        6786                 :          * In order for all trailing bytes to be zeroed, the caller needs to
        6787                 :          * use calloc(), followed by recalloc().  However, the current calloc()
        6788                 :          * implementation only zeros the bytes requested, so if recalloc() is
        6789                 :          * to work 100% correctly, calloc() will need to change to zero
        6790                 :          * trailing bytes.
        6791                 :          */
        6792                 :
        6793                 :         ptr = realloc(ptr, newsize);
        6794                 :         if (ptr != NULL && oldsize < newsize) {
        6795                 :                 memset((void *)((uintptr_t)ptr + oldsize), 0, newsize -
        6796                 :                     oldsize);
        6797                 :         }
        6798                 :
        6799                 :         return ptr;
        6800                 : }
        6801                 :
        6802                 : /*
        6803                 :  * This impl of _expand doesn't ever actually expand or shrink blocks: it
        6804                 :  * simply replies that you may continue using a shrunk block.
        6805                 :  */
        6806                 : void*
        6807                 : _expand(void *ptr, size_t newsize)
        6808                 : {
        6809                 :         if (isalloc(ptr) >= newsize)
        6810                 :                 return ptr;
        6811                 :
        6812                 :         return NULL;
        6813                 : }
        6814                 :
        6815                 : size_t
        6816                 : _msize(const void *ptr)
        6817                 : {
        6818                 :
        6819                 :         return malloc_usable_size(ptr);
        6820                 : }
        6821                 : #endif
        6822                 :
        6823                 : /*
        6824                 :  * End non-standard functions.
        6825                 :  */
        6826                 : /******************************************************************************/
        6827                 : /*
        6828                 :  * Begin library-private functions, used by threading libraries for protection
        6829                 :  * of malloc during fork().  These functions are only called if the program is
        6830                 :  * running in threaded mode, so there is no need to check whether the program
        6831                 :  * is threaded here.
        6832                 :  */
        6833                 :
        6834                 : static void
        6835               0 : _malloc_prefork(void)
        6836                 : {
        6837                 :         unsigned i;
        6838                 :
        6839                 :         /* Acquire all mutexes in a safe order. */
        6840                 :
        6841               0 :         malloc_spin_lock(&arenas_lock);
        6842               0 :         for (i = 0; i < narenas; i++) {
        6843               0 :                 if (arenas[i] != NULL)
        6844               0 :                         malloc_spin_lock(&arenas[i]->lock);
        6845                 :         }
        6846                 :
        6847               0 :         malloc_mutex_lock(&base_mtx);
        6848                 :
        6849               0 :         malloc_mutex_lock(&huge_mtx);
        6850               0 : }
        6851                 :
        6852                 : static void
        6853               0 : _malloc_postfork(void)
        6854                 : {
        6855                 :         unsigned i;
        6856                 :
        6857                 :         /* Release all mutexes, now that fork() has completed. */
        6858                 :
        6859               0 :         malloc_mutex_unlock(&huge_mtx);
        6860                 :
        6861               0 :         malloc_mutex_unlock(&base_mtx);
        6862                 :
        6863               0 :         for (i = 0; i < narenas; i++) {
        6864               0 :                 if (arenas[i] != NULL)
        6865               0 :                         malloc_spin_unlock(&arenas[i]->lock);
        6866                 :         }
        6867               0 :         malloc_spin_unlock(&arenas_lock);
        6868               0 : }
        6869                 :
        6870                 : /*
        6871                 :  * End library-private functions.
        6872                 :  */
        6873                 : /******************************************************************************/
        6874                 :
        6875                 : #ifdef HAVE_DLOPEN
        6876                 : #  include <dlfcn.h>
        6877                 : #endif
        6878                 :
        6879                 : #ifdef MOZ_MEMORY_DARWIN
        6880                 :
        6881                 : static void *
        6882                 : zone_malloc(malloc_zone_t *zone, size_t size)
        6883                 : {
        6884                 :
        6885                 :         return (malloc(size));
        6886                 : }
        6887                 :
        6888                 : static void *
        6889                 : zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
        6890                 : {
        6891                 :
        6892                 :         return (calloc(num, size));
        6893                 : }
        6894                 :
        6895                 : static void *
        6896                 : zone_valloc(malloc_zone_t *zone, size_t size)
        6897                 : {
        6898                 :         void *ret = NULL; /* Assignment avoids useless compiler warning. */
        6899                 :
        6900                 :         posix_memalign(&ret, pagesize, size);
        6901                 :
        6902                 :         return (ret);
        6903                 : }
        6904                 :
        6905                 : static void *
        6906                 : zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
        6907                 : {
        6908                 :         return (memalign(alignment, size));
        6909                 : }
        6910                 :
        6911                 : static void *
        6912                 : zone_destroy(malloc_zone_t *zone)
        6913                 : {
        6914                 :
        6915                 :         /* This function should never be called. */
        6916                 :         assert(false);
        6917                 :         return (NULL);
        6918                 : }
        6919                 :
        6920                 : static size_t
        6921                 : zone_good_size(malloc_zone_t *zone, size_t size)
        6922                 : {
        6923                 :         return je_malloc_good_size(size);
        6924                 : }
        6925                 :
        6926                 : static size_t
        6927                 : ozone_size(malloc_zone_t *zone, void *ptr)
        6928                 : {
        6929                 :         size_t ret = isalloc_validate(ptr);
        6930                 :         if (ret == 0)
        6931                 :                 ret = szone->size(zone, ptr);
        6932                 :
        6933                 :         return ret;
        6934                 : }
        6935                 :
        6936                 : static void
        6937                 : ozone_free(malloc_zone_t *zone, void *ptr)
        6938                 : {
        6939                 :         if (isalloc_validate(ptr) != 0)
        6940                 :                 free(ptr);
        6941                 :         else {
        6942                 :                 size_t size = szone->size(zone, ptr);
        6943                 :                 if (size != 0)
        6944                 :                         (szone->free)(zone, ptr);
        6945                 :                 /* Otherwise we leak. */
        6946                 :         }
        6947                 : }
        6948                 :
        6949                 : static void *
        6950                 : ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
        6951                 : {
        6952                 :     size_t oldsize;
        6953                 :         if (ptr == NULL)
        6954                 :                 return (malloc(size));
        6955                 :
        6956                 :         oldsize = isalloc_validate(ptr);
        6957                 :         if (oldsize != 0)
        6958                 :                 return (realloc(ptr, size));
        6959                 :         else {
        6960                 :                 oldsize = szone->size(zone, ptr);
        6961                 :                 if (oldsize == 0)
        6962                 :                         return (malloc(size));
        6963                 :                 else {
        6964                 :                         void *ret = malloc(size);
        6965                 :                         if (ret != NULL) {
        6966                 :                                 memcpy(ret, ptr, (oldsize < size) ? oldsize :
        6967                 :                                     size);
        6968                 :                                 (szone->free)(zone, ptr);
        6969                 :                         }
        6970                 :                         return (ret);
        6971                 :                 }
        6972                 :         }
        6973                 : }
        6974                 :
        6975                 : static unsigned
        6976                 : ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
        6977                 :     unsigned num_requested)
        6978                 : {
        6979                 :         /* Don't bother implementing this interface, since it isn't required. */
        6980                 :         return 0;
        6981                 : }
        6982                 :
        6983                 : static void
        6984                 : ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
        6985                 : {
        6986                 :         unsigned i;
        6987                 :
        6988                 :         for (i = 0; i < num; i++)
        6989                 :                 ozone_free(zone, to_be_freed[i]);
        6990                 : }
        6991                 :
        6992                 : static void
        6993                 : ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
        6994                 : {
        6995                 :         if (isalloc_validate(ptr) != 0) {
        6996                 :                 assert(isalloc_validate(ptr) == size);
        6997                 :                 free(ptr);
        6998                 :         } else {
        6999                 :                 assert(size == szone->size(zone, ptr));
        7000                 :                 l_szone.m16(zone, ptr, size);
        7001                 :         }
        7002                 : }
        7003                 :
        7004                 : static void
        7005                 : ozone_force_lock(malloc_zone_t *zone)
        7006                 : {
        7007                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
        7008                 :         szone->introspect->force_lock(zone);
        7009                 : }
        7010                 :
        7011                 : static void
        7012                 : ozone_force_unlock(malloc_zone_t *zone)
        7013                 : {
        7014                 :         /* jemalloc locking is taken care of by the normal jemalloc zone. */
        7015                 :         szone->introspect->force_unlock(zone);
        7016                 : }
        7017                 :
        7018                 : static size_t
        7019                 : zone_version_size(int version)
        7020                 : {
        7021                 :     switch (version)
        7022                 :     {
        7023                 :         case SNOW_LEOPARD_MALLOC_ZONE_T_VERSION:
        7024                 :             return sizeof(snow_leopard_malloc_zone);
        7025                 :         case LEOPARD_MALLOC_ZONE_T_VERSION:
        7026                 :             return sizeof(leopard_malloc_zone);
        7027                 :         default:
        7028                 :         case LION_MALLOC_ZONE_T_VERSION:
        7029                 :             return sizeof(lion_malloc_zone);
        7030                 :     }
        7031                 : }
        7032                 :
        7033                 : /*
        7034                 :  * Overlay the default scalable zone (szone) such that existing allocations are
        7035                 :  * drained, and further allocations come from jemalloc. This is necessary
        7036                 :  * because Core Foundation directly accesses and uses the szone before the
        7037                 :  * jemalloc library is even loaded.
        7038                 :  */
        7039                 : static void
        7040                 : szone2ozone(malloc_zone_t *default_zone, size_t size)
        7041                 : {
        7042                 :     lion_malloc_zone *l_zone;
        7043                 :         assert(malloc_initialized);
        7044                 :
        7045                 :         /*
        7046                 :          * Stash a copy of the original szone so that we can call its
        7047                 :          * functions as needed. Note that internally, the szone stores its
        7048                 :          * bookkeeping data structures immediately following the malloc_zone_t
        7049                 :          * header, so when calling szone functions, we need to pass a pointer to
        7050                 :          * the original zone structure.
        7051                 :          */
        7052                 :         memcpy(szone, default_zone, size);
        7053                 :
        7054                 :         /* OSX 10.7 allocates the default zone in protected memory. */
        7055                 :         if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
        7056                 :                 void* start_of_page = (void*)((size_t)(default_zone) & ~pagesize_mask);
        7057                 :                 mprotect (start_of_page, size, PROT_READ | PROT_WRITE);
        7058                 :         }
        7059                 :
        7060                 :         default_zone->size = (void *)ozone_size;
        7061                 :         default_zone->malloc = (void *)zone_malloc;
        7062                 :         default_zone->calloc = (void *)zone_calloc;
        7063                 :         default_zone->valloc = (void *)zone_valloc;
        7064                 :         default_zone->free = (void *)ozone_free;
        7065                 :         default_zone->realloc = (void *)ozone_realloc;
        7066                 :         default_zone->destroy = (void *)zone_destroy;
        7067                 :         default_zone->batch_malloc = NULL;
        7068                 :         default_zone->batch_free = ozone_batch_free;
        7069                 :         default_zone->introspect = ozone_introspect;
        7070                 :
        7071                 :         /* Don't modify default_zone->zone_name; Mac libc may rely on the name
        7072                 :          * being unchanged.  See Mozilla bug 694896. */
        7073                 :
        7074                 :         ozone_introspect->enumerator = NULL;
        7075                 :         ozone_introspect->good_size = (void *)zone_good_size;
        7076                 :         ozone_introspect->check = NULL;
        7077                 :         ozone_introspect->print = NULL;
        7078                 :         ozone_introspect->log = NULL;
        7079                 :         ozone_introspect->force_lock = (void *)ozone_force_lock;
        7080                 :         ozone_introspect->force_unlock = (void *)ozone_force_unlock;
        7081                 :         ozone_introspect->statistics = NULL;
        7082                 :
        7083                 :     /* Platform-dependent structs */
        7084                 :     l_zone = (lion_malloc_zone*)(default_zone);
        7085                 :
        7086                 :     if (default_zone->version >= SNOW_LEOPARD_MALLOC_ZONE_T_VERSION) {
        7087                 :         l_zone->m15 = (void (*)())zone_memalign;
        7088                 :         l_zone->m16 = (void (*)())ozone_free_definite_size;
        7089                 :         l_ozone_introspect.m9 = NULL;
        7090                 :     }
        7091                 :
        7092                 :     if (default_zone->version >= LION_MALLOC_ZONE_T_VERSION) {
        7093                 :         l_zone->m17 = NULL;
        7094                 :         l_ozone_introspect.m10 = NULL;
        7095                 :         l_ozone_introspect.m11 = NULL;
        7096                 :         l_ozone_introspect.m12 = NULL;
        7097                 :         l_ozone_introspect.m13 = NULL;
        7098                 :     }
        7099                 : }
        7100                 :
        7101                 : __attribute__((constructor))
        7102                 : void
        7103                 : jemalloc_darwin_init(void)
        7104                 : {
        7105                 :         if (malloc_init_hard())
        7106                 :                 abort();
        7107                 : }
        7108                 :
        7109                 : #elif defined(__GLIBC__) && !defined(__UCLIBC__)
        7110                 : /*
        7111                 :  * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible
        7112                 :  * to inconsistently reference libc's malloc(3)-compatible functions
        7113                 :  * (bug 493541).
        7114                 :  *
        7115                 :  * These definitions interpose hooks in glibc.  The functions are actually
        7116                 :  * passed an extra argument for the caller return address, which will be
        7117                 :  * ignored.
        7118                 :  */
        7119                 : void (*__free_hook)(void *ptr) = free;
        7120                 : void *(*__malloc_hook)(size_t size) = malloc;
        7121                 : void *(*__realloc_hook)(void *ptr, size_t size) = realloc;
        7122                 : void *(*__memalign_hook)(size_t alignment, size_t size) = MEMALIGN;
        7123                 :
        7124                 : #elif defined(RTLD_DEEPBIND)
        7125                 : /*
        7126                 :  * XXX On systems that support RTLD_GROUP or DF_1_GROUP, do their
        7127                 :  * implementations permit similar inconsistencies?  Should STV_SINGLETON
        7128                 :  * visibility be used for interposition where available?
        7129                 :  */
        7130                 : #  error "Interposing malloc is unsafe on this system without libc malloc hooks."
        7131                 : #endif
        7132                 :
        7133                 : #ifdef MOZ_MEMORY_WINDOWS
        7134                 : /*
        7135                 :  * In the new style jemalloc integration jemalloc is built as a separate
        7136                 :  * shared library.  Since we're no longer hooking into the CRT binary,
        7137                 :  * we need to initialize the heap at the first opportunity we get.
        7138                 :  * DLL_PROCESS_ATTACH in DllMain is that opportunity.
        7139                 :  */
        7140                 : BOOL APIENTRY DllMain(HINSTANCE hModule,
        7141                 :                       DWORD reason,
        7142                 :                       LPVOID lpReserved)
        7143                 : {
        7144                 :   switch (reason) {
        7145                 :     case DLL_PROCESS_ATTACH:
        7146                 :       /* Don't force the system to page DllMain back in every time
        7147                 :        * we create/destroy a thread */
        7148                 :       DisableThreadLibraryCalls(hModule);
        7149                 :       /* Initialize the heap */
        7150                 :       malloc_init_hard();
        7151                 :       break;
        7152                 :
        7153                 :     case DLL_PROCESS_DETACH:
        7154                 :       break;
        7155                 :
        7156                 :   }
        7157                 :
        7158                 :   return TRUE;
        7159                 : }
        7160                 :
        7161                 : /*
        7162                 :  *  There's a fun allocator mismatch in (at least) the VS 2010 CRT
        7163                 :  *  (see the giant comment in this directory's Makefile.in
        7164                 :  *  that gets redirected here to avoid a crash on shutdown.
        7165                 :  */
        7166                 : void
        7167                 : je_dumb_free_thunk(void *ptr)
        7168                 : {
        7169                 :   return; /* shutdown leaks that we don't care about */
        7170                 : }
        7171                 :
        7172                 : #include <wchar.h>
        7173                 :
        7174                 : /*
        7175                 :  *  We also need to provide our own impl of wcsdup so that we don't ask
        7176                 :  *  the CRT for memory from its heap (which will then be unfreeable).
        7177                 :  */
        7178                 : wchar_t *je_wcsdup(const wchar_t *src)
        7179                 : {
        7180                 :   size_t len = wcslen(src);
        7181                 :   wchar_t* dst = (wchar_t*)je_malloc((len + 1) * sizeof(wchar_t));
        7182                 :   if(dst)
        7183                 :     wcsncpy(dst, src, len + 1);
        7184                 :   return dst;
        7185                 : }
        7186                 :
        7187                 : #endif

Generated by: LCOV version 1.7