LCOV - app.info - js/src/jsgc.cpp

LCOV - code coverage report
| Current view: | directory - js/src - jsgc.cpp (source / functions) |  | Found | Hit | Coverage |
| Test: | app.info | Lines: | 2016 | 1571 | 77.9 %
| Date: | 2012-04-07 | Functions: | 205 | 164 | 80.0 %

           1                 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
           2                 :  * vim: set ts=8 sw=4 et tw=78:
           3                 :  *
           4                 :  * ***** BEGIN LICENSE BLOCK *****
           5                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
           6                 :  *
           7                 :  * The contents of this file are subject to the Mozilla Public License Version
           8                 :  * 1.1 (the "License"); you may not use this file except in compliance with
           9                 :  * the License. You may obtain a copy of the License at
          10                 :  * http://www.mozilla.org/MPL/
          11                 :  *
          12                 :  * Software distributed under the License is distributed on an "AS IS" basis,
          13                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
          14                 :  * for the specific language governing rights and limitations under the
          15                 :  * License.
          16                 :  *
          17                 :  * The Original Code is Mozilla Communicator client code, released
          18                 :  * March 31, 1998.
          19                 :  *
          20                 :  * The Initial Developer of the Original Code is
          21                 :  * Netscape Communications Corporation.
          22                 :  * Portions created by the Initial Developer are Copyright (C) 1998
          23                 :  * the Initial Developer. All Rights Reserved.
          24                 :  *
          25                 :  * Contributor(s):
          26                 :  *
          27                 :  * Alternatively, the contents of this file may be used under the terms of
          28                 :  * either of the GNU General Public License Version 2 or later (the "GPL"),
          29                 :  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
          30                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
          31                 :  * of those above. If you wish to allow use of your version of this file only
          32                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
          33                 :  * use your version of this file under the terms of the MPL, indicate your
          34                 :  * decision by deleting the provisions above and replace them with the notice
          35                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
          36                 :  * the provisions above, a recipient may use your version of this file under
          37                 :  * the terms of any one of the MPL, the GPL or the LGPL.
          38                 :  *
          39                 :  * ***** END LICENSE BLOCK ***** */
          40                 :
          41                 : /* JS Mark-and-Sweep Garbage Collector. */
          42                 :
          43                 : #include "mozilla/Attributes.h"
          44                 : #include "mozilla/Util.h"
          45                 :
          46                 : /*
          47                 :  * This code implements a mark-and-sweep garbage collector. The mark phase is
          48                 :  * incremental. Most sweeping is done on a background thread. A GC is divided
          49                 :  * into slices as follows:
          50                 :  *
          51                 :  * Slice 1: Roots pushed onto the mark stack. The mark stack is processed by
          52                 :  * popping an element, marking it, and pushing its children.
          53                 :  *   ... JS code runs ...
          54                 :  * Slice 2: More mark stack processing.
          55                 :  *   ... JS code runs ...
          56                 :  * Slice n-1: More mark stack processing.
          57                 :  *   ... JS code runs ...
          58                 :  * Slice n: Mark stack is completely drained. Some sweeping is done.
          59                 :  *   ... JS code runs, remaining sweeping done on background thread ...
          60                 :  *
          61                 :  * When background sweeping finishes the GC is complete.
          62                 :  *
          63                 :  * Incremental GC requires close collaboration with the mutator (i.e., JS code):
          64                 :  *
          65                 :  * 1. During an incremental GC, if a memory location (except a root) is written
          66                 :  * to, then the value it previously held must be marked. Write barriers ensure
          67                 :  * this.
          68                 :  * 2. Any object that is allocated during incremental GC must start out marked.
          69                 :  * 3. Roots are special memory locations that don't need write
          70                 :  * barriers. However, they must be marked in the first slice. Roots are things
          71                 :  * like the C stack and the VM stack, since it would be too expensive to put
          72                 :  * barriers on them.
          73                 :  */
          74                 :
          75                 : #include <math.h>
          76                 : #include <string.h>     /* for memset used when DEBUG */
          77                 :
          78                 : #include "jstypes.h"
          79                 : #include "jsutil.h"
          80                 : #include "jshash.h"
          81                 : #include "jsclist.h"
          82                 : #include "jsprf.h"
          83                 : #include "jsapi.h"
          84                 : #include "jsatom.h"
          85                 : #include "jscompartment.h"
          86                 : #include "jscrashreport.h"
          87                 : #include "jscrashformat.h"
          88                 : #include "jscntxt.h"
          89                 : #include "jsversion.h"
          90                 : #include "jsdbgapi.h"
          91                 : #include "jsexn.h"
          92                 : #include "jsfun.h"
          93                 : #include "jsgc.h"
          94                 : #include "jsgcmark.h"
          95                 : #include "jsinterp.h"
          96                 : #include "jsiter.h"
          97                 : #include "jslock.h"
          98                 : #include "jsnum.h"
          99                 : #include "jsobj.h"
         100                 : #include "jsprobes.h"
         101                 : #include "jsproxy.h"
         102                 : #include "jsscope.h"
         103                 : #include "jsscript.h"
         104                 : #include "jswatchpoint.h"
         105                 : #include "jsweakmap.h"
         106                 : #if JS_HAS_XML_SUPPORT
         107                 : #include "jsxml.h"
         108                 : #endif
         109                 :
         110                 : #include "frontend/Parser.h"
         111                 : #include "gc/Memory.h"
         112                 : #include "methodjit/MethodJIT.h"
         113                 : #include "vm/Debugger.h"
         114                 : #include "vm/String.h"
         115                 :
         116                 : #include "jsinterpinlines.h"
         117                 : #include "jsobjinlines.h"
         118                 :
         119                 : #include "vm/ScopeObject-inl.h"
         120                 : #include "vm/String-inl.h"
         121                 :
         122                 : #ifdef MOZ_VALGRIND
         123                 : # define JS_VALGRIND
         124                 : #endif
         125                 : #ifdef JS_VALGRIND
         126                 : # include <valgrind/memcheck.h>
         127                 : #endif
         128                 :
         129                 : #ifdef XP_WIN
         130                 : # include "jswin.h"
         131                 : #else
         132                 : # include <unistd.h>
         133                 : #endif
         134                 :
         135                 : using namespace mozilla;
         136                 : using namespace js;
         137                 : using namespace js::gc;
         138                 :
         139                 : namespace js {
         140                 : namespace gc {
         141                 :
         142                 : /*
         143                 :  * Lower limit after which we limit the heap growth
         144                 :  */
         145                 : const size_t GC_ALLOCATION_THRESHOLD = 30 * 1024 * 1024;
         146                 :
         147                 : /*
         148                 :  * A GC is triggered once the number of newly allocated arenas is
         149                 :  * GC_HEAP_GROWTH_FACTOR times the number of live arenas after the last GC
         150                 :  * starting after the lower limit of GC_ALLOCATION_THRESHOLD. This number is
         151                 :  * used for non-incremental GCs.
         152                 :  */
         153                 : const float GC_HEAP_GROWTH_FACTOR = 3.0f;
         154                 :
         155                 : /* Perform a Full GC every 20 seconds if MaybeGC is called */
         156                 : static const uint64_t GC_IDLE_FULL_SPAN = 20 * 1000 * 1000;
         157                 :
         158                 : #ifdef JS_GC_ZEAL
         159                 : static void
         160                 : StartVerifyBarriers(JSContext *cx);
         161                 :
         162                 : static void
         163                 : EndVerifyBarriers(JSContext *cx);
         164                 :
         165                 : void
         166                 : FinishVerifier(JSRuntime *rt);
         167                 : #endif
         168                 :
         169                 : /* This array should be const, but that doesn't link right under GCC. */
         170                 : AllocKind slotsToThingKind[] = {
         171                 :     /* 0 */  FINALIZE_OBJECT0,  FINALIZE_OBJECT2,  FINALIZE_OBJECT2,  FINALIZE_OBJECT4,
         172                 :     /* 4 */  FINALIZE_OBJECT4,  FINALIZE_OBJECT8,  FINALIZE_OBJECT8,  FINALIZE_OBJECT8,
         173                 :     /* 8 */  FINALIZE_OBJECT8,  FINALIZE_OBJECT12, FINALIZE_OBJECT12, FINALIZE_OBJECT12,
         174                 :     /* 12 */ FINALIZE_OBJECT12, FINALIZE_OBJECT16, FINALIZE_OBJECT16, FINALIZE_OBJECT16,
         175                 :     /* 16 */ FINALIZE_OBJECT16
         176                 : };
         177                 :
         178                 : JS_STATIC_ASSERT(JS_ARRAY_LENGTH(slotsToThingKind) == SLOTS_TO_THING_KIND_LIMIT);
         179                 :
         180                 : const uint32_t Arena::ThingSizes[] = {
         181                 :     sizeof(JSObject),           /* FINALIZE_OBJECT0             */
         182                 :     sizeof(JSObject),           /* FINALIZE_OBJECT0_BACKGROUND  */
         183                 :     sizeof(JSObject_Slots2),    /* FINALIZE_OBJECT2             */
         184                 :     sizeof(JSObject_Slots2),    /* FINALIZE_OBJECT2_BACKGROUND  */
         185                 :     sizeof(JSObject_Slots4),    /* FINALIZE_OBJECT4             */
         186                 :     sizeof(JSObject_Slots4),    /* FINALIZE_OBJECT4_BACKGROUND  */
         187                 :     sizeof(JSObject_Slots8),    /* FINALIZE_OBJECT8             */
         188                 :     sizeof(JSObject_Slots8),    /* FINALIZE_OBJECT8_BACKGROUND  */
         189                 :     sizeof(JSObject_Slots12),   /* FINALIZE_OBJECT12            */
         190                 :     sizeof(JSObject_Slots12),   /* FINALIZE_OBJECT12_BACKGROUND */
         191                 :     sizeof(JSObject_Slots16),   /* FINALIZE_OBJECT16            */
         192                 :     sizeof(JSObject_Slots16),   /* FINALIZE_OBJECT16_BACKGROUND */
         193                 :     sizeof(JSScript),           /* FINALIZE_SCRIPT              */
         194                 :     sizeof(Shape),              /* FINALIZE_SHAPE               */
         195                 :     sizeof(BaseShape),          /* FINALIZE_BASE_SHAPE          */
         196                 :     sizeof(types::TypeObject),  /* FINALIZE_TYPE_OBJECT         */
         197                 : #if JS_HAS_XML_SUPPORT
         198                 :     sizeof(JSXML),              /* FINALIZE_XML                 */
         199                 : #endif
         200                 :     sizeof(JSShortString),      /* FINALIZE_SHORT_STRING        */
         201                 :     sizeof(JSString),           /* FINALIZE_STRING              */
         202                 :     sizeof(JSExternalString),   /* FINALIZE_EXTERNAL_STRING     */
         203                 : };
         204                 :
         205                 : #define OFFSET(type) uint32_t(sizeof(ArenaHeader) + (ArenaSize - sizeof(ArenaHeader)) % sizeof(type))
         206                 :
         207                 : const uint32_t Arena::FirstThingOffsets[] = {
         208                 :     OFFSET(JSObject),           /* FINALIZE_OBJECT0             */
         209                 :     OFFSET(JSObject),           /* FINALIZE_OBJECT0_BACKGROUND  */
         210                 :     OFFSET(JSObject_Slots2),    /* FINALIZE_OBJECT2             */
         211                 :     OFFSET(JSObject_Slots2),    /* FINALIZE_OBJECT2_BACKGROUND  */
         212                 :     OFFSET(JSObject_Slots4),    /* FINALIZE_OBJECT4             */
         213                 :     OFFSET(JSObject_Slots4),    /* FINALIZE_OBJECT4_BACKGROUND  */
         214                 :     OFFSET(JSObject_Slots8),    /* FINALIZE_OBJECT8             */
         215                 :     OFFSET(JSObject_Slots8),    /* FINALIZE_OBJECT8_BACKGROUND  */
         216                 :     OFFSET(JSObject_Slots12),   /* FINALIZE_OBJECT12            */
         217                 :     OFFSET(JSObject_Slots12),   /* FINALIZE_OBJECT12_BACKGROUND */
         218                 :     OFFSET(JSObject_Slots16),   /* FINALIZE_OBJECT16            */
         219                 :     OFFSET(JSObject_Slots16),   /* FINALIZE_OBJECT16_BACKGROUND */
         220                 :     OFFSET(JSScript),           /* FINALIZE_SCRIPT              */
         221                 :     OFFSET(Shape),              /* FINALIZE_SHAPE               */
         222                 :     OFFSET(BaseShape),          /* FINALIZE_BASE_SHAPE          */
         223                 :     OFFSET(types::TypeObject),  /* FINALIZE_TYPE_OBJECT         */
         224                 : #if JS_HAS_XML_SUPPORT
         225                 :     OFFSET(JSXML),              /* FINALIZE_XML                 */
         226                 : #endif
         227                 :     OFFSET(JSShortString),      /* FINALIZE_SHORT_STRING        */
         228                 :     OFFSET(JSString),           /* FINALIZE_STRING              */
         229                 :     OFFSET(JSExternalString),   /* FINALIZE_EXTERNAL_STRING     */
         230                 : };
         231                 :
         232                 : #undef OFFSET
         233                 :
         234                 : #ifdef DEBUG
         235                 : void
         236        12401769 : ArenaHeader::checkSynchronizedWithFreeList() const
         237                 : {
         238                 :     /*
         239                 :      * Do not allow to access the free list when its real head is still stored
         240                 :      * in FreeLists and is not synchronized with this one.
         241                 :      */
         242        12401769 :     JS_ASSERT(allocated());
         243                 :
         244                 :     /*
         245                 :      * We can be called from the background finalization thread when the free
         246                 :      * list in the compartment can mutate at any moment. We cannot do any
         247                 :      * checks in this case.
         248                 :      */
         249        12401663 :     if (!compartment->rt->gcRunning)
         250         1115994 :         return;
         251                 :
         252        11285669 :     FreeSpan firstSpan = FreeSpan::decodeOffsets(arenaAddress(), firstFreeSpanOffsets);
         253        11285669 :     if (firstSpan.isEmpty())
         254         1361529 :         return;
         255         9924140 :     const FreeSpan *list = compartment->arenas.getFreeList(getAllocKind());
         256         9924140 :     if (list->isEmpty() || firstSpan.arenaAddress() != list->arenaAddress())
         257         9136469 :         return;
         258                 :
         259                 :     /*
         260                 :      * Here this arena has free things, FreeList::lists[thingKind] is not
         261                 :      * empty and also points to this arena. Thus they must the same.
         262                 :      */
         263          787671 :     JS_ASSERT(firstSpan.isSameNonEmptySpan(list));
         264                 : }
         265                 : #endif
         266                 :
         267                 : /* static */ void
         268               0 : Arena::staticAsserts()
         269                 : {
         270                 :     JS_STATIC_ASSERT(sizeof(Arena) == ArenaSize);
         271                 :     JS_STATIC_ASSERT(JS_ARRAY_LENGTH(ThingSizes) == FINALIZE_LIMIT);
         272                 :     JS_STATIC_ASSERT(JS_ARRAY_LENGTH(FirstThingOffsets) == FINALIZE_LIMIT);
         273               0 : }
         274                 :
         275                 : template<typename T>
         276                 : inline bool
         277         2955046 : Arena::finalize(FreeOp *fop, AllocKind thingKind, size_t thingSize)
         278                 : {
         279                 :     /* Enforce requirements on size of T. */
         280         2955046 :     JS_ASSERT(thingSize % Cell::CellSize == 0);
         281         2955046 :     JS_ASSERT(thingSize <= 255);
         282                 :
         283         2955046 :     JS_ASSERT(aheader.allocated());
         284         2955046 :     JS_ASSERT(thingKind == aheader.getAllocKind());
         285         2955046 :     JS_ASSERT(thingSize == aheader.getThingSize());
         286         2955046 :     JS_ASSERT(!aheader.hasDelayedMarking);
         287         2955046 :     JS_ASSERT(!aheader.markOverflow);
         288         2955046 :     JS_ASSERT(!aheader.allocatedDuringIncremental);
         289                 :
         290         2955046 :     uintptr_t thing = thingsStart(thingKind);
         291         2955046 :     uintptr_t lastByte = thingsEnd() - 1;
         292                 :
         293         2955046 :     FreeSpan nextFree(aheader.getFirstFreeSpan());
         294         2955046 :     nextFree.checkSpan();
         295                 :
         296         2955046 :     FreeSpan newListHead;
         297         2955046 :     FreeSpan *newListTail = &newListHead;
         298         2955046 :     uintptr_t newFreeSpanStart = 0;
         299         2955046 :     bool allClear = true;
         300         5910092 :     DebugOnly<size_t> nmarked = 0;
         301       382256469 :     for (;; thing += thingSize) {
         302       385211515 :         JS_ASSERT(thing <= lastByte + 1);
         303       385211515 :         if (thing == nextFree.first) {
         304         4804812 :             JS_ASSERT(nextFree.last <= lastByte);
         305         4804812 :             if (nextFree.last == lastByte)
         306                 :                 break;
         307         1849766 :             JS_ASSERT(Arena::isAligned(nextFree.last, thingSize));
         308         1849766 :             if (!newFreeSpanStart)
         309         1468285 :                 newFreeSpanStart = thing;
         310         1849766 :             thing = nextFree.last;
         311         1849766 :             nextFree = *nextFree.nextSpan();
         312         1849766 :             nextFree.checkSpan();
         313                 :         } else {
         314       380406703 :             T *t = reinterpret_cast<T *>(thing);
         315       380406703 :             if (t->isMarked()) {
         316       248839476 :                 allClear = false;
         317       248839476 :                 nmarked++;
         318       248839476 :                 if (newFreeSpanStart) {
         319         2397087 :                     JS_ASSERT(thing >= thingsStart(thingKind) + thingSize);
         320         2397087 :                     newListTail->first = newFreeSpanStart;
         321         2397087 :                     newListTail->last = thing - thingSize;
         322         2397087 :                     newListTail = newListTail->nextSpanUnchecked(thingSize);
         323         2397087 :                     newFreeSpanStart = 0;
         324                 :                 }
         325                 :             } else {
         326       131567227 :                 if (!newFreeSpanStart)
         327         2316803 :                     newFreeSpanStart = thing;
         328       131567227 :                 t->finalize(fop);
         329       131567227 :                 JS_POISON(t, JS_FREE_PATTERN, thingSize);
         330                 :             }
         331                 :         }
         332                 :     }
         333                 :
         334         2955046 :     if (allClear) {
         335         1326848 :         JS_ASSERT(newListTail == &newListHead);
         336         1326848 :         JS_ASSERT(newFreeSpanStart == thingsStart(thingKind));
         337         1326848 :         return true;
         338                 :     }
         339                 :
         340         1628198 :     newListTail->first = newFreeSpanStart ? newFreeSpanStart : nextFree.first;
         341         1628198 :     JS_ASSERT(Arena::isAligned(newListTail->first, thingSize));
         342         1628198 :     newListTail->last = lastByte;
         343                 :
         344                 : #ifdef DEBUG
         345         1628198 :     size_t nfree = 0;
         346         4025285 :     for (const FreeSpan *span = &newListHead; span != newListTail; span = span->nextSpan()) {
         347         2397087 :         span->checkSpan();
         348         2397087 :         JS_ASSERT(Arena::isAligned(span->first, thingSize));
         349         2397087 :         JS_ASSERT(Arena::isAligned(span->last, thingSize));
         350         2397087 :         nfree += (span->last - span->first) / thingSize + 1;
         351         2397087 :         JS_ASSERT(nfree + nmarked <= thingsPerArena(thingSize));
         352                 :     }
         353         1628198 :     nfree += (newListTail->last + 1 - newListTail->first) / thingSize;
         354         1628198 :     JS_ASSERT(nfree + nmarked == thingsPerArena(thingSize));
         355                 : #endif
         356         1628198 :     aheader.setFirstFreeSpan(&newListHead);
         357                 :
         358         1628198 :     return false;
         359                 : }
         360                 :
         361                 : template<typename T>
         362                 : inline void
         363         1318324 : FinalizeTypedArenas(FreeOp *fop, ArenaLists::ArenaList *al, AllocKind thingKind)
         364                 : {
         365                 :     /*
         366                 :      * Release empty arenas and move non-full arenas with some free things into
         367                 :      * a separated list that we append to al after the loop to ensure that any
         368                 :      * arena before al->cursor is full.
         369                 :      */
         370         1318324 :     JS_ASSERT_IF(!al->head, al->cursor == &al->head);
         371         1318324 :     ArenaLists::ArenaList available;
         372         1318324 :     ArenaHeader **ap = &al->head;
         373         1318324 :     size_t thingSize = Arena::thingSize(thingKind);
         374         7228416 :     while (ArenaHeader *aheader = *ap) {
         375         2955046 :         bool allClear = aheader->getArena()->finalize<T>(fop, thingKind, thingSize);
         376         2955046 :         if (allClear) {
         377         1326848 :             *ap = aheader->next;
         378         1326848 :             aheader->chunk()->releaseArena(aheader);
         379         1628198 :         } else if (aheader->hasFreeThings()) {
         380          705613 :             *ap = aheader->next;
         381          705613 :             *available.cursor = aheader;
         382          705613 :             available.cursor = &aheader->next;
         383                 :         } else {
         384          922585 :             ap = &aheader->next;
         385                 :         }
         386                 :     }
         387                 :
         388                 :     /* Terminate the available list and append it to al. */
         389         1318324 :     *available.cursor = NULL;
         390         1318324 :     *ap = available.head;
         391         1318324 :     al->cursor = ap;
         392         2274962 :     JS_ASSERT_IF(!al->head, al->cursor == &al->head);
         393         1318324 : }
         394                 :
         395                 : /*
         396                 :  * Finalize the list. On return al->cursor points to the first non-empty arena
         397                 :  * after the al->head.
         398                 :  */
         399                 : static void
         400         1318324 : FinalizeArenas(FreeOp *fop, ArenaLists::ArenaList *al, AllocKind thingKind)
         401                 : {
         402         1318324 :     switch(thingKind) {
         403                 :       case FINALIZE_OBJECT0:
         404                 :       case FINALIZE_OBJECT0_BACKGROUND:
         405                 :       case FINALIZE_OBJECT2:
         406                 :       case FINALIZE_OBJECT2_BACKGROUND:
         407                 :       case FINALIZE_OBJECT4:
         408                 :       case FINALIZE_OBJECT4_BACKGROUND:
         409                 :       case FINALIZE_OBJECT8:
         410                 :       case FINALIZE_OBJECT8_BACKGROUND:
         411                 :       case FINALIZE_OBJECT12:
         412                 :       case FINALIZE_OBJECT12_BACKGROUND:
         413                 :       case FINALIZE_OBJECT16:
         414                 :       case FINALIZE_OBJECT16_BACKGROUND:
         415          688867 :         FinalizeTypedArenas<JSObject>(fop, al, thingKind);
         416          688867 :         break;
         417                 :       case FINALIZE_SCRIPT:
         418           83697 :         FinalizeTypedArenas<JSScript>(fop, al, thingKind);
         419           83697 :         break;
         420                 :       case FINALIZE_SHAPE:
         421           83697 :         FinalizeTypedArenas<Shape>(fop, al, thingKind);
         422           83697 :         break;
         423                 :       case FINALIZE_BASE_SHAPE:
         424           83697 :         FinalizeTypedArenas<BaseShape>(fop, al, thingKind);
         425           83697 :         break;
         426                 :       case FINALIZE_TYPE_OBJECT:
         427           83697 :         FinalizeTypedArenas<types::TypeObject>(fop, al, thingKind);
         428           83697 :         break;
         429                 : #if JS_HAS_XML_SUPPORT
         430                 :       case FINALIZE_XML:
         431           83697 :         FinalizeTypedArenas<JSXML>(fop, al, thingKind);
         432           83697 :         break;
         433                 : #endif
         434                 :       case FINALIZE_STRING:
         435           83224 :         FinalizeTypedArenas<JSString>(fop, al, thingKind);
         436           83224 :         break;
         437                 :       case FINALIZE_SHORT_STRING:
         438           44051 :         FinalizeTypedArenas<JSShortString>(fop, al, thingKind);
         439           44051 :         break;
         440                 :       case FINALIZE_EXTERNAL_STRING:
         441           83697 :         FinalizeTypedArenas<JSExternalString>(fop, al, thingKind);
         442           83697 :         break;
         443                 :     }
         444         1318324 : }
         445                 :
         446                 : static inline Chunk *
         447           40491 : AllocChunk() {
         448           40491 :     return static_cast<Chunk *>(MapAlignedPages(ChunkSize, ChunkSize));
         449                 : }
         450                 :
         451                 : static inline void
         452           40491 : FreeChunk(Chunk *p) {
         453           40491 :     UnmapPages(static_cast<void *>(p), ChunkSize);
         454           40491 : }
         455                 :
         456                 : #ifdef JS_THREADSAFE
         457                 : inline bool
         458           42904 : ChunkPool::wantBackgroundAllocation(JSRuntime *rt) const
         459                 : {
         460                 :     /*
         461                 :      * To minimize memory waste we do not want to run the background chunk
         462                 :      * allocation if we have empty chunks or when the runtime needs just few
         463                 :      * of them.
         464                 :      */
         465           42904 :     return rt->gcHelperThread.canBackgroundAllocate() &&
         466                 :            emptyCount == 0 &&
         467           42904 :            rt->gcChunkSet.count() >= 4;
         468                 : }
         469                 : #endif
         470                 :
         471                 : /* Must be called with the GC lock taken. */
         472                 : inline Chunk *
         473           40393 : ChunkPool::get(JSRuntime *rt)
         474                 : {
         475           40393 :     JS_ASSERT(this == &rt->gcChunkPool);
         476                 :
         477           40393 :     Chunk *chunk = emptyChunkListHead;
         478           40393 :     if (chunk) {
         479            2413 :         JS_ASSERT(emptyCount);
         480            2413 :         emptyChunkListHead = chunk->info.next;
         481            2413 :         --emptyCount;
         482                 :     } else {
         483           37980 :         JS_ASSERT(!emptyCount);
         484           37980 :         chunk = Chunk::allocate(rt);
         485           37980 :         if (!chunk)
         486               0 :             return NULL;
         487           37980 :         JS_ASSERT(chunk->info.numArenasFreeCommitted == ArenasPerChunk);
         488           37980 :         rt->gcNumArenasFreeCommitted += ArenasPerChunk;
         489                 :     }
         490           40393 :     JS_ASSERT(chunk->unused());
         491           40393 :     JS_ASSERT(!rt->gcChunkSet.has(chunk));
         492                 :
         493                 : #ifdef JS_THREADSAFE
         494           40393 :     if (wantBackgroundAllocation(rt))
         495            2511 :         rt->gcHelperThread.startBackgroundAllocationIfIdle();
         496                 : #endif
         497                 :
         498           40393 :     return chunk;
         499                 : }
         500                 :
         501                 : /* Must be called either during the GC or with the GC lock taken. */
         502                 : inline void
         503           42904 : ChunkPool::put(Chunk *chunk)
         504                 : {
         505           42904 :     chunk->info.age = 0;
         506           42904 :     chunk->info.next = emptyChunkListHead;
         507           42904 :     emptyChunkListHead = chunk;
         508           42904 :     emptyCount++;
         509           42904 : }
         510                 :
         511                 : /* Must be called either during the GC or with the GC lock taken. */
         512                 : Chunk *
         513           38436 : ChunkPool::expire(JSRuntime *rt, bool releaseAll)
         514                 : {
         515           38436 :     JS_ASSERT(this == &rt->gcChunkPool);
         516                 :
         517                 :     /*
         518                 :      * Return old empty chunks to the system while preserving the order of
         519                 :      * other chunks in the list. This way, if the GC runs several times
         520                 :      * without emptying the list, the older chunks will stay at the tail
         521                 :      * and are more likely to reach the max age.
         522                 :      */
         523           38436 :     Chunk *freeList = NULL;
         524          118057 :     for (Chunk **chunkp = &emptyChunkListHead; *chunkp; ) {
         525           41185 :         JS_ASSERT(emptyCount);
         526           41185 :         Chunk *chunk = *chunkp;
         527           41185 :         JS_ASSERT(chunk->unused());
         528           41185 :         JS_ASSERT(!rt->gcChunkSet.has(chunk));
         529           41185 :         JS_ASSERT(chunk->info.age <= MAX_EMPTY_CHUNK_AGE);
         530           41185 :         if (releaseAll || chunk->info.age == MAX_EMPTY_CHUNK_AGE) {
         531           40491 :             *chunkp = chunk->info.next;
         532           40491 :             --emptyCount;
         533           40491 :             chunk->prepareToBeFreed(rt);
         534           40491 :             chunk->info.next = freeList;
         535           40491 :             freeList = chunk;
         536                 :         } else {
         537                 :             /* Keep the chunk but increase its age. */
         538             694 :             ++chunk->info.age;
         539             694 :             chunkp = &chunk->info.next;
         540                 :         }
         541                 :     }
         542           38436 :     JS_ASSERT_IF(releaseAll, !emptyCount);
         543           38436 :     return freeList;
         544                 : }
         545                 :
         546                 : static void
         547           59261 : FreeChunkList(Chunk *chunkListHead)
         548                 : {
         549           99752 :     while (Chunk *chunk = chunkListHead) {
         550           40491 :         JS_ASSERT(!chunk->info.numArenasFreeCommitted);
         551           40491 :         chunkListHead = chunk->info.next;
         552           40491 :         FreeChunk(chunk);
         553                 :     }
         554           18770 : }
         555                 :
         556                 : void
         557           18761 : ChunkPool::expireAndFree(JSRuntime *rt, bool releaseAll)
         558                 : {
         559           18761 :     FreeChunkList(expire(rt, releaseAll));
         560           18761 : }
         561                 :
         562                 : JS_FRIEND_API(int64_t)
         563               0 : ChunkPool::countCleanDecommittedArenas(JSRuntime *rt)
         564                 : {
         565               0 :     JS_ASSERT(this == &rt->gcChunkPool);
         566                 :
         567               0 :     int64_t numDecommitted = 0;
         568               0 :     Chunk *chunk = emptyChunkListHead;
         569               0 :     while (chunk) {
         570               0 :         for (uint32_t i = 0; i < ArenasPerChunk; ++i)
         571               0 :             if (chunk->decommittedArenas.get(i))
         572               0 :                 ++numDecommitted;
         573               0 :         chunk = chunk->info.next;
         574                 :     }
         575               0 :     return numDecommitted;
         576                 : }
         577                 :
         578                 : /* static */ Chunk *
         579           40491 : Chunk::allocate(JSRuntime *rt)
         580                 : {
         581           40491 :     Chunk *chunk = static_cast<Chunk *>(AllocChunk());
         582           40491 :     if (!chunk)
         583               0 :         return NULL;
         584           40491 :     chunk->init();
         585           40491 :     rt->gcStats.count(gcstats::STAT_NEW_CHUNK);
         586           40491 :     return chunk;
         587                 : }
         588                 :
         589                 : /* Must be called with the GC lock taken. */
         590                 : /* static */ inline void
         591               0 : Chunk::release(JSRuntime *rt, Chunk *chunk)
         592                 : {
         593               0 :     JS_ASSERT(chunk);
         594               0 :     chunk->prepareToBeFreed(rt);
         595               0 :     FreeChunk(chunk);
         596               0 : }
         597                 :
         598                 : inline void
         599           40491 : Chunk::prepareToBeFreed(JSRuntime *rt)
         600                 : {
         601           40491 :     JS_ASSERT(rt->gcNumArenasFreeCommitted >= info.numArenasFreeCommitted);
         602           40491 :     rt->gcNumArenasFreeCommitted -= info.numArenasFreeCommitted;
         603           40491 :     rt->gcStats.count(gcstats::STAT_DESTROY_CHUNK);
         604                 :
         605                 : #ifdef DEBUG
         606                 :     /*
         607                 :      * Let FreeChunkList detect a missing prepareToBeFreed call before it
         608                 :      * frees chunk.
         609                 :      */
         610           40491 :     info.numArenasFreeCommitted = 0;
         611                 : #endif
         612           40491 : }
         613                 :
         614                 : void
         615           40491 : Chunk::init()
         616                 : {
         617           40491 :     JS_POISON(this, JS_FREE_PATTERN, ChunkSize);
         618                 :
         619                 :     /*
         620                 :      * We clear the bitmap to guard against xpc_IsGrayGCThing being called on
         621                 :      * uninitialized data, which would happen before the first GC cycle.
         622                 :      */
         623           40491 :     bitmap.clear();
         624                 :
         625                 :     /* Initialize the arena tracking bitmap. */
         626           40491 :     decommittedArenas.clear(false);
         627                 :
         628                 :     /* Initialize the chunk info. */
         629           40491 :     info.freeArenasHead = &arenas[0].aheader;
         630           40491 :     info.lastDecommittedArenaOffset = 0;
         631           40491 :     info.numArenasFree = ArenasPerChunk;
         632           40491 :     info.numArenasFreeCommitted = ArenasPerChunk;
         633           40491 :     info.age = 0;
         634                 :
         635                 :     /* Initialize the arena header state. */
         636        10244223 :     for (unsigned i = 0; i < ArenasPerChunk; i++) {
         637        10203732 :         arenas[i].aheader.setAsNotAllocated();
         638                 :         arenas[i].aheader.next = (i + 1 < ArenasPerChunk)
         639                 :                                  ? &arenas[i + 1].aheader
         640        10203732 :                                  : NULL;
         641                 :     }
         642                 :
         643                 :     /* The rest of info fields are initialized in PickChunk. */
         644           40491 : }
         645                 :
         646                 : inline Chunk **
         647         1726847 : GetAvailableChunkList(JSCompartment *comp)
         648                 : {
         649         1726847 :     JSRuntime *rt = comp->rt;
         650                 :     return comp->isSystemCompartment
         651                 :            ? &rt->gcSystemAvailableChunkListHead
         652         1726847 :            : &rt->gcUserAvailableChunkListHead;
         653                 : }
         654                 :
         655                 : inline void
         656           43479 : Chunk::addToAvailableList(JSCompartment *comp)
         657                 : {
         658           43479 :     insertToAvailableList(GetAvailableChunkList(comp));
         659           43479 : }
         660                 :
         661                 : inline void
         662           43479 : Chunk::insertToAvailableList(Chunk **insertPoint)
         663                 : {
         664           43479 :     JS_ASSERT(hasAvailableArenas());
         665           43479 :     JS_ASSERT(!info.prevp);
         666           43479 :     JS_ASSERT(!info.next);
         667           43479 :     info.prevp = insertPoint;
         668           43479 :     Chunk *insertBefore = *insertPoint;
         669           43479 :     if (insertBefore) {
         670            3085 :         JS_ASSERT(insertBefore->info.prevp == insertPoint);
         671            3085 :         insertBefore->info.prevp = &info.next;
         672                 :     }
         673           43479 :     info.next = insertBefore;
         674           43479 :     *insertPoint = this;
         675           43479 : }
         676                 :
         677                 : inline void
         678           43479 : Chunk::removeFromAvailableList()
         679                 : {
         680           43479 :     JS_ASSERT(info.prevp);
         681           43479 :     *info.prevp = info.next;
         682           43479 :     if (info.next) {
         683            2847 :         JS_ASSERT(info.next->info.prevp == &info.next);
         684            2847 :         info.next->info.prevp = info.prevp;
         685                 :     }
         686           43479 :     info.prevp = NULL;
         687           43479 :     info.next = NULL;
         688           43479 : }
         689                 :
         690                 : /*
         691                 :  * Search for and return the next decommitted Arena. Our goal is to keep
         692                 :  * lastDecommittedArenaOffset "close" to a free arena. We do this by setting
         693                 :  * it to the most recently freed arena when we free, and forcing it to
         694                 :  * the last alloc + 1 when we allocate.
         695                 :  */
         696                 : uint32_t
         697               0 : Chunk::findDecommittedArenaOffset()
         698                 : {
         699                 :     /* Note: lastFreeArenaOffset can be past the end of the list. */
         700               0 :     for (unsigned i = info.lastDecommittedArenaOffset; i < ArenasPerChunk; i++)
         701               0 :         if (decommittedArenas.get(i))
         702               0 :             return i;
         703               0 :     for (unsigned i = 0; i < info.lastDecommittedArenaOffset; i++)
         704               0 :         if (decommittedArenas.get(i))
         705               0 :             return i;
         706               0 :     JS_NOT_REACHED("No decommitted arenas found.");
         707                 :     return -1;
         708                 : }
         709                 :
         710                 : ArenaHeader *
         711               0 : Chunk::fetchNextDecommittedArena()
         712                 : {
         713               0 :     JS_ASSERT(info.numArenasFreeCommitted == 0);
         714               0 :     JS_ASSERT(info.numArenasFree > 0);
         715                 :
         716               0 :     unsigned offset = findDecommittedArenaOffset();
         717               0 :     info.lastDecommittedArenaOffset = offset + 1;
         718               0 :     --info.numArenasFree;
         719               0 :     decommittedArenas.unset(offset);
         720                 :
         721               0 :     Arena *arena = &arenas[offset];
         722               0 :     MarkPagesInUse(arena, ArenaSize);
         723               0 :     arena->aheader.setAsNotAllocated();
         724                 :
         725               0 :     return &arena->aheader;
         726                 : }
         727                 :
         728                 : inline ArenaHeader *
         729         1687458 : Chunk::fetchNextFreeArena(JSRuntime *rt)
         730                 : {
         731         1687458 :     JS_ASSERT(info.numArenasFreeCommitted > 0);
         732         1687458 :     JS_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree);
         733         1687458 :     JS_ASSERT(info.numArenasFreeCommitted <= rt->gcNumArenasFreeCommitted);
         734                 :
         735         1687458 :     ArenaHeader *aheader = info.freeArenasHead;
         736         1687458 :     info.freeArenasHead = aheader->next;
         737         1687458 :     --info.numArenasFreeCommitted;
         738         1687458 :     --info.numArenasFree;
         739         1687458 :     --rt->gcNumArenasFreeCommitted;
         740                 :
         741         1687458 :     return aheader;
         742                 : }
         743                 :
         744                 : ArenaHeader *
         745         1683368 : Chunk::allocateArena(JSCompartment *comp, AllocKind thingKind)
         746                 : {
         747         1683368 :     JS_ASSERT(hasAvailableArenas());
         748                 :
         749         1683368 :     JSRuntime *rt = comp->rt;
         750         1683368 :     JS_ASSERT(rt->gcBytes <= rt->gcMaxBytes);
         751         1683368 :     if (rt->gcMaxBytes - rt->gcBytes < ArenaSize)
         752              41 :         return NULL;
         753                 :
         754         1683327 :     ArenaHeader *aheader = JS_LIKELY(info.numArenasFreeCommitted > 0)
         755                 :                            ? fetchNextFreeArena(rt)
         756         1683327 :                            : fetchNextDecommittedArena();
         757         1683327 :     aheader->init(comp, thingKind);
         758         1683327 :     if (JS_UNLIKELY(!hasAvailableArenas()))
         759            3086 :         removeFromAvailableList();
         760                 :
         761         1683327 :     Probes::resizeHeap(comp, rt->gcBytes, rt->gcBytes + ArenaSize);
         762         1683327 :     rt->gcBytes += ArenaSize;
         763         1683327 :     comp->gcBytes += ArenaSize;
         764         1683327 :     if (comp->gcBytes >= comp->gcTriggerBytes)
         765           97581 :         TriggerCompartmentGC(comp, gcreason::ALLOC_TRIGGER);
         766                 :
         767         1683327 :     return aheader;
         768                 : }
         769                 :
         770                 : inline void
         771         1683327 : Chunk::addArenaToFreeList(JSRuntime *rt, ArenaHeader *aheader)
         772                 : {
         773         1683327 :     JS_ASSERT(!aheader->allocated());
         774         1683327 :     aheader->next = info.freeArenasHead;
         775         1683327 :     info.freeArenasHead = aheader;
         776         1683327 :     ++info.numArenasFreeCommitted;
         777         1683327 :     ++info.numArenasFree;
         778         1683327 :     ++rt->gcNumArenasFreeCommitted;
         779         1683327 : }
         780                 :
         781                 : void
         782         1683327 : Chunk::releaseArena(ArenaHeader *aheader)
         783                 : {
         784         1683327 :     JS_ASSERT(aheader->allocated());
         785         1683327 :     JS_ASSERT(!aheader->hasDelayedMarking);
         786         1683327 :     JSCompartment *comp = aheader->compartment;
         787         1683327 :     JSRuntime *rt = comp->rt;
         788                 : #ifdef JS_THREADSAFE
         789         3366654 :     AutoLockGC maybeLock;
         790         1683327 :     if (rt->gcHelperThread.sweeping())
         791          220528 :         maybeLock.lock(rt);
         792                 : #endif
         793                 :
         794         1683327 :     Probes::resizeHeap(comp, rt->gcBytes, rt->gcBytes - ArenaSize);
         795         1683327 :     JS_ASSERT(rt->gcBytes >= ArenaSize);
         796         1683327 :     JS_ASSERT(comp->gcBytes >= ArenaSize);
         797                 : #ifdef JS_THREADSAFE
         798         1683327 :     if (rt->gcHelperThread.sweeping())
         799          220528 :         comp->reduceGCTriggerBytes(GC_HEAP_GROWTH_FACTOR * ArenaSize);
         800                 : #endif
         801         1683327 :     rt->gcBytes -= ArenaSize;
         802         1683327 :     comp->gcBytes -= ArenaSize;
         803                 :
         804         1683327 :     aheader->setAsNotAllocated();
         805         1683327 :     addArenaToFreeList(rt, aheader);
         806                 :
         807         1683327 :     if (info.numArenasFree == 1) {
         808            3086 :         JS_ASSERT(!info.prevp);
         809            3086 :         JS_ASSERT(!info.next);
         810            3086 :         addToAvailableList(comp);
         811         1680241 :     } else if (!unused()) {
         812         1639848 :         JS_ASSERT(info.prevp);
         813                 :     } else {
         814           40393 :         rt->gcChunkSet.remove(this);
         815           40393 :         removeFromAvailableList();
         816           40393 :         rt->gcChunkPool.put(this);
         817                 :     }
         818         1683327 : }
         819                 :
         820                 : } /* namespace gc */
         821                 : } /* namespace js */
         822                 :
         823                 : /* The caller must hold the GC lock. */
         824                 : static Chunk *
         825         1683368 : PickChunk(JSCompartment *comp)
         826                 : {
         827         1683368 :     JSRuntime *rt = comp->rt;
         828         1683368 :     Chunk **listHeadp = GetAvailableChunkList(comp);
         829         1683368 :     Chunk *chunk = *listHeadp;
         830         1683368 :     if (chunk)
         831         1642975 :         return chunk;
         832                 :
         833           40393 :     chunk = rt->gcChunkPool.get(rt);
         834           40393 :     if (!chunk)
         835               0 :         return NULL;
         836                 :
         837           40393 :     rt->gcChunkAllocationSinceLastGC = true;
         838                 :
         839                 :     /*
         840                 :      * FIXME bug 583732 - chunk is newly allocated and cannot be present in
         841                 :      * the table so using ordinary lookupForAdd is suboptimal here.
         842                 :      */
         843           80786 :     GCChunkSet::AddPtr p = rt->gcChunkSet.lookupForAdd(chunk);
         844           40393 :     JS_ASSERT(!p);
         845           40393 :     if (!rt->gcChunkSet.add(p, chunk)) {
         846               0 :         Chunk::release(rt, chunk);
         847               0 :         return NULL;
         848                 :     }
         849                 :
         850           40393 :     chunk->info.prevp = NULL;
         851           40393 :     chunk->info.next = NULL;
         852           40393 :     chunk->addToAvailableList(comp);
         853                 :
         854           40393 :     return chunk;
         855                 : }
         856                 :
         857                 : JS_FRIEND_API(bool)
         858        23655060 : IsAboutToBeFinalized(const Cell *thing)
         859                 : {
         860        23655060 :     JSCompartment *thingCompartment = reinterpret_cast<const Cell *>(thing)->compartment();
         861        23655060 :     if (!thingCompartment->isCollecting())
         862           87469 :         return false;
         863        23567591 :     return !reinterpret_cast<const Cell *>(thing)->isMarked();
         864                 : }
         865                 :
         866                 : bool
         867          167880 : IsAboutToBeFinalized(const Value &v)
         868                 : {
         869          167880 :     JS_ASSERT(v.isMarkable());
         870          167880 :     return IsAboutToBeFinalized((Cell *)v.toGCThing());
         871                 : }
         872                 :
         873                 : /* Lifetime for type sets attached to scripts containing observed types. */
         874                 : static const int64_t JIT_SCRIPT_RELEASE_TYPES_INTERVAL = 60 * 1000 * 1000;
         875                 :
         876                 : JSBool
         877           18761 : js_InitGC(JSRuntime *rt, uint32_t maxbytes)
         878                 : {
         879           18761 :     if (!rt->gcChunkSet.init(INITIAL_CHUNK_CAPACITY))
         880               0 :         return false;
         881                 :
         882           18761 :     if (!rt->gcRootsHash.init(256))
         883               0 :         return false;
         884                 :
         885           18761 :     if (!rt->gcLocksHash.init(256))
         886               0 :         return false;
         887                 :
         888                 : #ifdef JS_THREADSAFE
         889           18761 :     rt->gcLock = PR_NewLock();
         890           18761 :     if (!rt->gcLock)
         891               0 :         return false;
         892           18761 :     if (!rt->gcHelperThread.init())
         893               0 :         return false;
         894                 : #endif
         895                 :
         896                 :     /*
         897                 :      * Separate gcMaxMallocBytes from gcMaxBytes but initialize to maxbytes
         898                 :      * for default backward API compatibility.
         899                 :      */
         900           18761 :     rt->gcMaxBytes = maxbytes;
         901           18761 :     rt->setGCMaxMallocBytes(maxbytes);
         902                 :
         903           18761 :     rt->gcJitReleaseTime = PRMJ_Now() + JIT_SCRIPT_RELEASE_TYPES_INTERVAL;
         904           18761 :     return true;
         905                 : }
         906                 :
         907                 : namespace js {
         908                 :
         909                 : inline bool
         910        11157038 : InFreeList(ArenaHeader *aheader, uintptr_t addr)
         911                 : {
         912        11157038 :     if (!aheader->hasFreeThings())
         913         2194159 :         return false;
         914                 :
         915         8962879 :     FreeSpan firstSpan(aheader->getFirstFreeSpan());
         916                 :
         917        18237028 :     for (const FreeSpan *span = &firstSpan;;) {
         918                 :         /* If the thing comes fore the current span, it's not free. */
         919        18237028 :         if (addr < span->first)
         920         8961983 :             return false;
         921                 :
         922                 :         /*
         923                 :          * If we find it inside the span, it's dead. We use here "<=" and not
         924                 :          * "<" even for the last span as we know that thing is inside the
         925                 :          * arena. Thus for the last span thing < span->end.
         926                 :          */
         927         9275045 :         if (addr <= span->last)
         928             896 :             return true;
         929                 :
         930                 :         /*
         931                 :          * The last possible empty span is an the end of the arena. Here
         932                 :          * span->end < thing < thingsEnd and so we must have more spans.
         933                 :          */
         934         9274149 :         span = span->nextSpan();
         935                 :     }
         936                 : }
         937                 :
         938                 : enum ConservativeGCTest
         939                 : {
         940                 :     CGCT_VALID,
         941                 :     CGCT_LOWBITSET, /* excluded because one of the low bits was set */
         942                 :     CGCT_NOTARENA,  /* not within arena range in a chunk */
         943                 :     CGCT_OTHERCOMPARTMENT,  /* in another compartment */
         944                 :     CGCT_NOTCHUNK,  /* not within a valid chunk */
         945                 :     CGCT_FREEARENA, /* within arena containing only free things */
         946                 :     CGCT_NOTLIVE,   /* gcthing is not allocated */
         947                 :     CGCT_END
         948                 : };
         949                 :
         950                 : /*
         951                 :  * Tests whether w is a (possibly dead) GC thing. Returns CGCT_VALID and
         952                 :  * details about the thing if so. On failure, returns the reason for rejection.
         953                 :  */
         954                 : inline ConservativeGCTest
         955       247550640 : IsAddressableGCThing(JSRuntime *rt, uintptr_t w,
         956                 :                      gc::AllocKind *thingKindPtr, ArenaHeader **arenaHeader, void **thing)
         957                 : {
         958                 :     /*
         959                 :      * We assume that the compiler never uses sub-word alignment to store
         960                 :      * pointers and does not tag pointers on its own. Additionally, the value
         961                 :      * representation for all values and the jsid representation for GC-things
         962                 :      * do not touch the low two bits. Thus any word with the low two bits set
         963                 :      * is not a valid GC-thing.
         964                 :      */
         965                 :     JS_STATIC_ASSERT(JSID_TYPE_STRING == 0 && JSID_TYPE_OBJECT == 4);
         966       247550640 :     if (w & 0x3)
         967        40978656 :         return CGCT_LOWBITSET;
         968                 :
         969                 :     /*
         970                 :      * An object jsid has its low bits tagged. In the value representation on
         971                 :      * 64-bit, the high bits are tagged.
         972                 :      */
         973       206571984 :     const uintptr_t JSID_PAYLOAD_MASK = ~uintptr_t(JSID_TYPE_MASK);
         974                 : #if JS_BITS_PER_WORD == 32
         975       206571984 :     uintptr_t addr = w & JSID_PAYLOAD_MASK;
         976                 : #elif JS_BITS_PER_WORD == 64
         977                 :     uintptr_t addr = w & JSID_PAYLOAD_MASK & JSVAL_PAYLOAD_MASK;
         978                 : #endif
         979                 :
         980       206571984 :     Chunk *chunk = Chunk::fromAddress(addr);
         981                 :
         982       206571984 :     if (!rt->gcChunkSet.has(chunk))
         983       195012761 :         return CGCT_NOTCHUNK;
         984                 :
         985                 :     /*
         986                 :      * We query for pointers outside the arena array after checking for an
         987                 :      * allocated chunk. Such pointers are rare and we want to reject them
         988                 :      * after doing more likely rejections.
         989                 :      */
         990        11559223 :     if (!Chunk::withinArenasRange(addr))
         991            7436 :         return CGCT_NOTARENA;
         992                 :
         993                 :     /* If the arena is not currently allocated, don't access the header. */
         994        11551787 :     size_t arenaOffset = Chunk::arenaIndex(addr);
         995        11551787 :     if (chunk->decommittedArenas.get(arenaOffset))
         996               0 :         return CGCT_FREEARENA;
         997                 :
         998        11551787 :     ArenaHeader *aheader = &chunk->arenas[arenaOffset].aheader;
         999                 :
        1000        11551787 :     if (!aheader->allocated())
        1001               2 :         return CGCT_FREEARENA;
        1002                 :
        1003        11551785 :     if (rt->gcRunning && !aheader->compartment->isCollecting())
        1004          221252 :         return CGCT_OTHERCOMPARTMENT;
        1005                 :
        1006        11330533 :     AllocKind thingKind = aheader->getAllocKind();
        1007        11330533 :     uintptr_t offset = addr & ArenaMask;
        1008        11330533 :     uintptr_t minOffset = Arena::firstThingOffset(thingKind);
        1009        11330533 :     if (offset < minOffset)
        1010          173495 :         return CGCT_NOTARENA;
        1011                 :
        1012                 :     /* addr can point inside the thing so we must align the address. */
        1013        11157038 :     uintptr_t shift = (offset - minOffset) % Arena::thingSize(thingKind);
        1014        11157038 :     addr -= shift;
        1015                 :
        1016        11157038 :     if (thing)
        1017        11157038 :         *thing = reinterpret_cast<void *>(addr);
        1018        11157038 :     if (arenaHeader)
        1019        11157038 :         *arenaHeader = aheader;
        1020        11157038 :     if (thingKindPtr)
        1021        11157038 :         *thingKindPtr = thingKind;
        1022        11157038 :     return CGCT_VALID;
        1023                 : }
        1024                 :
        1025                 : /*
        1026                 :  * Returns CGCT_VALID and mark it if the w can be a  live GC thing and sets
        1027                 :  * thingKind accordingly. Otherwise returns the reason for rejection.
        1028                 :  */
        1029                 : inline ConservativeGCTest
        1030       247550640 : MarkIfGCThingWord(JSTracer *trc, uintptr_t w)
        1031                 : {
        1032                 :     void *thing;
        1033                 :     ArenaHeader *aheader;
        1034                 :     AllocKind thingKind;
        1035       247550640 :     ConservativeGCTest status = IsAddressableGCThing(trc->runtime, w, &thingKind, &aheader, &thing);
        1036       247550640 :     if (status != CGCT_VALID)
        1037       236393602 :         return status;
        1038                 :
        1039                 :     /*
        1040                 :      * Check if the thing is free. We must use the list of free spans as at
        1041                 :      * this point we no longer have the mark bits from the previous GC run and
        1042                 :      * we must account for newly allocated things.
        1043                 :      */
        1044        11157038 :     if (InFreeList(aheader, uintptr_t(thing)))
        1045             896 :         return CGCT_NOTLIVE;
        1046                 :
        1047        11156142 :     JSGCTraceKind traceKind = MapAllocToTraceKind(thingKind);
        1048                 : #ifdef DEBUG
        1049        11156142 :     const char pattern[] = "machine_stack %p";
        1050                 :     char nameBuf[sizeof(pattern) - 2 + sizeof(thing) * 2];
        1051        11156142 :     JS_snprintf(nameBuf, sizeof(nameBuf), pattern, thing);
        1052        11156142 :     JS_SET_TRACING_NAME(trc, nameBuf);
        1053                 : #endif
        1054        11156142 :     void *tmp = thing;
        1055        11156142 :     MarkKind(trc, &tmp, traceKind);
        1056        11156142 :     JS_ASSERT(tmp == thing);
        1057                 :
        1058                 : #ifdef DEBUG
        1059        11156142 :     if (trc->runtime->gcIncrementalState == MARK_ROOTS)
        1060             466 :         trc->runtime->gcSavedRoots.append(JSRuntime::SavedGCRoot(thing, traceKind));
        1061                 : #endif
        1062                 :
        1063        11156142 :     return CGCT_VALID;
        1064                 : }
        1065                 :
        1066                 : static void
        1067       247550640 : MarkWordConservatively(JSTracer *trc, uintptr_t w)
        1068                 : {
        1069                 :     /*
        1070                 :      * The conservative scanner may access words that valgrind considers as
        1071                 :      * undefined. To avoid false positives and not to alter valgrind view of
        1072                 :      * the memory we make as memcheck-defined the argument, a copy of the
        1073                 :      * original word. See bug 572678.
        1074                 :      */
        1075                 : #ifdef JS_VALGRIND
        1076                 :     JS_SILENCE_UNUSED_VALUE_IN_EXPR(VALGRIND_MAKE_MEM_DEFINED(&w, sizeof(w)));
        1077                 : #endif
        1078                 :
        1079       247550640 :     MarkIfGCThingWord(trc, w);
        1080       247550640 : }
        1081                 :
        1082                 : static void
        1083           43304 : MarkRangeConservatively(JSTracer *trc, const uintptr_t *begin, const uintptr_t *end)
        1084                 : {
        1085           43304 :     JS_ASSERT(begin <= end);
        1086       247593944 :     for (const uintptr_t *i = begin; i < end; ++i)
        1087       247550640 :         MarkWordConservatively(trc, *i);
        1088           43304 : }
        1089                 :
        1090                 : static JS_NEVER_INLINE void
        1091           22875 : MarkConservativeStackRoots(JSTracer *trc, bool useSavedRoots)
        1092                 : {
        1093           22875 :     JSRuntime *rt = trc->runtime;
        1094                 :
        1095                 : #ifdef DEBUG
        1096           22875 :     if (useSavedRoots) {
        1097            3378 :         for (JSRuntime::SavedGCRoot *root = rt->gcSavedRoots.begin();
        1098            1689 :              root != rt->gcSavedRoots.end();
        1099                 :              root++)
        1100                 :         {
        1101             466 :             JS_SET_TRACING_NAME(trc, "cstack");
        1102             466 :             MarkKind(trc, &root->thing, root->kind);
        1103                 :         }
        1104            1223 :         return;
        1105                 :     }
        1106                 :
        1107           21652 :     if (rt->gcIncrementalState == MARK_ROOTS)
        1108            1446 :         rt->gcSavedRoots.clearAndFree();
        1109                 : #endif
        1110                 :
        1111           21652 :     ConservativeGCData *cgcd = &rt->conservativeGC;
        1112           21652 :     if (!cgcd->hasStackToScan()) {
        1113                 : #ifdef JS_THREADSAFE
        1114               0 :         JS_ASSERT(!rt->suspendCount);
        1115               0 :         JS_ASSERT(rt->requestDepth <= cgcd->requestThreshold);
        1116                 : #endif
        1117               0 :         return;
        1118                 :     }
        1119                 :
        1120                 :     uintptr_t *stackMin, *stackEnd;
        1121                 : #if JS_STACK_GROWTH_DIRECTION > 0
        1122                 :     stackMin = rt->nativeStackBase;
        1123                 :     stackEnd = cgcd->nativeStackTop;
        1124                 : #else
        1125           21652 :     stackMin = cgcd->nativeStackTop + 1;
        1126           21652 :     stackEnd = reinterpret_cast<uintptr_t *>(rt->nativeStackBase);
        1127                 : #endif
        1128                 :
        1129           21652 :     JS_ASSERT(stackMin <= stackEnd);
        1130           21652 :     MarkRangeConservatively(trc, stackMin, stackEnd);
        1131                 :     MarkRangeConservatively(trc, cgcd->registerSnapshot.words,
        1132           21652 :                             ArrayEnd(cgcd->registerSnapshot.words));
        1133                 : }
        1134                 :
        1135                 : void
        1136               0 : MarkStackRangeConservatively(JSTracer *trc, Value *beginv, Value *endv)
        1137                 : {
        1138               0 :     const uintptr_t *begin = beginv->payloadWord();
        1139               0 :     const uintptr_t *end = endv->payloadWord();
        1140                 : #ifdef JS_NUNBOX32
        1141                 :     /*
        1142                 :      * With 64-bit jsvals on 32-bit systems, we can optimize a bit by
        1143                 :      * scanning only the payloads.
        1144                 :      */
        1145               0 :     JS_ASSERT(begin <= end);
        1146               0 :     for (const uintptr_t *i = begin; i < end; i += sizeof(Value) / sizeof(uintptr_t))
        1147               0 :         MarkWordConservatively(trc, *i);
        1148                 : #else
        1149                 :     MarkRangeConservatively(trc, begin, end);
        1150                 : #endif
        1151               0 : }
        1152                 :
        1153                 :
        1154                 :
        1155                 : JS_NEVER_INLINE void
        1156           23149 : ConservativeGCData::recordStackTop()
        1157                 : {
        1158                 :     /* Update the native stack pointer if it points to a bigger stack. */
        1159                 :     uintptr_t dummy;
        1160           23149 :     nativeStackTop = &dummy;
        1161                 :
        1162                 :     /*
        1163                 :      * To record and update the register snapshot for the conservative scanning
        1164                 :      * with the latest values we use setjmp.
        1165                 :      */
        1166                 : #if defined(_MSC_VER)
        1167                 : # pragma warning(push)
        1168                 : # pragma warning(disable: 4611)
        1169                 : #endif
        1170           23149 :     (void) setjmp(registerSnapshot.jmpbuf);
        1171                 : #if defined(_MSC_VER)
        1172                 : # pragma warning(pop)
        1173                 : #endif
        1174           23149 : }
        1175                 :
        1176                 : static void
        1177           41950 : RecordNativeStackTopForGC(JSRuntime *rt)
        1178                 : {
        1179           41950 :     ConservativeGCData *cgcd = &rt->conservativeGC;
        1180                 :
        1181                 : #ifdef JS_THREADSAFE
        1182                 :     /* Record the stack top here only if we are called from a request. */
        1183           41950 :     JS_ASSERT(rt->requestDepth >= cgcd->requestThreshold);
        1184           41950 :     if (rt->requestDepth == cgcd->requestThreshold)
        1185           18961 :         return;
        1186                 : #endif
        1187           22989 :     cgcd->recordStackTop();
        1188                 : }
        1189                 :
        1190                 : } /* namespace js */
        1191                 :
        1192                 : bool
        1193               0 : js_IsAddressableGCThing(JSRuntime *rt, uintptr_t w, gc::AllocKind *thingKind, void **thing)
        1194                 : {
        1195               0 :     return js::IsAddressableGCThing(rt, w, thingKind, NULL, thing) == CGCT_VALID;
        1196                 : }
        1197                 :
        1198                 : #ifdef DEBUG
        1199                 : static void
        1200                 : CheckLeakedRoots(JSRuntime *rt);
        1201                 : #endif
        1202                 :
        1203                 : void
        1204           18761 : js_FinishGC(JSRuntime *rt)
        1205                 : {
        1206                 :     /*
        1207                 :      * Wait until the background finalization stops and the helper thread
        1208                 :      * shuts down before we forcefully release any remaining GC memory.
        1209                 :      */
        1210                 : #ifdef JS_THREADSAFE
        1211           18761 :     rt->gcHelperThread.finish();
        1212                 : #endif
        1213                 :
        1214                 : #ifdef JS_GC_ZEAL
        1215                 :     /* Free memory associated with GC verification. */
        1216           18761 :     FinishVerifier(rt);
        1217                 : #endif
        1218                 :
        1219                 :     /* Delete all remaining Compartments. */
        1220           37522 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        1221           18761 :         Foreground::delete_(c.get());
        1222           18761 :     rt->compartments.clear();
        1223           18761 :     rt->atomsCompartment = NULL;
        1224                 :
        1225           18761 :     rt->gcSystemAvailableChunkListHead = NULL;
        1226           18761 :     rt->gcUserAvailableChunkListHead = NULL;
        1227           18761 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
        1228               0 :         Chunk::release(rt, r.front());
        1229           18761 :     rt->gcChunkSet.clear();
        1230                 :
        1231           18761 :     rt->gcChunkPool.expireAndFree(rt, true);
        1232                 :
        1233                 : #ifdef DEBUG
        1234           18761 :     if (!rt->gcRootsHash.empty())
        1235               0 :         CheckLeakedRoots(rt);
        1236                 : #endif
        1237           18761 :     rt->gcRootsHash.clear();
        1238           18761 :     rt->gcLocksHash.clear();
        1239           18761 : }
        1240                 :
        1241                 : JSBool
        1242             171 : js_AddRoot(JSContext *cx, Value *vp, const char *name)
        1243                 : {
        1244             171 :     JSBool ok = js_AddRootRT(cx->runtime, vp, name);
        1245             171 :     if (!ok)
        1246               0 :         JS_ReportOutOfMemory(cx);
        1247             171 :     return ok;
        1248                 : }
        1249                 :
        1250                 : JSBool
        1251           18666 : js_AddGCThingRoot(JSContext *cx, void **rp, const char *name)
        1252                 : {
        1253           18666 :     JSBool ok = js_AddGCThingRootRT(cx->runtime, rp, name);
        1254           18666 :     if (!ok)
        1255               0 :         JS_ReportOutOfMemory(cx);
        1256           18666 :     return ok;
        1257                 : }
        1258                 :
        1259                 : JS_FRIEND_API(JSBool)
        1260             171 : js_AddRootRT(JSRuntime *rt, jsval *vp, const char *name)
        1261                 : {
        1262                 :     return !!rt->gcRootsHash.put((void *)vp,
        1263             171 :                                  RootInfo(name, JS_GC_ROOT_VALUE_PTR));
        1264                 : }
        1265                 :
        1266                 : JS_FRIEND_API(JSBool)
        1267           18666 : js_AddGCThingRootRT(JSRuntime *rt, void **rp, const char *name)
        1268                 : {
        1269                 :     return !!rt->gcRootsHash.put((void *)rp,
        1270           18666 :                                  RootInfo(name, JS_GC_ROOT_GCTHING_PTR));
        1271                 : }
        1272                 :
        1273                 : JS_FRIEND_API(void)
        1274           18837 : js_RemoveRoot(JSRuntime *rt, void *rp)
        1275                 : {
        1276           18837 :     rt->gcRootsHash.remove(rp);
        1277           18837 :     rt->gcPoke = true;
        1278           18837 : }
        1279                 :
        1280                 : typedef RootedValueMap::Range RootRange;
        1281                 : typedef RootedValueMap::Entry RootEntry;
        1282                 : typedef RootedValueMap::Enum RootEnum;
        1283                 :
        1284                 : #ifdef DEBUG
        1285                 :
        1286                 : static void
        1287               0 : CheckLeakedRoots(JSRuntime *rt)
        1288                 : {
        1289               0 :     uint32_t leakedroots = 0;
        1290                 :
        1291                 :     /* Warn (but don't assert) debug builds of any remaining roots. */
        1292               0 :     for (RootRange r = rt->gcRootsHash.all(); !r.empty(); r.popFront()) {
        1293               0 :         RootEntry &entry = r.front();
        1294               0 :         leakedroots++;
        1295                 :         fprintf(stderr,
        1296                 :                 "JS engine warning: leaking GC root \'%s\' at %p\n",
        1297               0 :                 entry.value.name ? entry.value.name : "", entry.key);
        1298                 :     }
        1299                 :
        1300               0 :     if (leakedroots > 0) {
        1301               0 :         if (leakedroots == 1) {
        1302                 :             fprintf(stderr,
        1303                 : "JS engine warning: 1 GC root remains after destroying the JSRuntime at %p.\n"
        1304                 : "                   This root may point to freed memory. Objects reachable\n"
        1305                 : "                   through it have not been finalized.\n",
        1306               0 :                     (void *) rt);
        1307                 :         } else {
        1308                 :             fprintf(stderr,
        1309                 : "JS engine warning: %lu GC roots remain after destroying the JSRuntime at %p.\n"
        1310                 : "                   These roots may point to freed memory. Objects reachable\n"
        1311                 : "                   through them have not been finalized.\n",
        1312               0 :                     (unsigned long) leakedroots, (void *) rt);
        1313                 :         }
        1314                 :     }
        1315               0 : }
        1316                 :
        1317                 : void
        1318               0 : js_DumpNamedRoots(JSRuntime *rt,
        1319                 :                   void (*dump)(const char *name, void *rp, JSGCRootType type, void *data),
        1320                 :                   void *data)
        1321                 : {
        1322               0 :     for (RootRange r = rt->gcRootsHash.all(); !r.empty(); r.popFront()) {
        1323               0 :         RootEntry &entry = r.front();
        1324               0 :         if (const char *name = entry.value.name)
        1325               0 :             dump(name, entry.key, entry.value.type, data);
        1326                 :     }
        1327               0 : }
        1328                 :
        1329                 : #endif /* DEBUG */
        1330                 :
        1331                 : uint32_t
        1332               0 : js_MapGCRoots(JSRuntime *rt, JSGCRootMapFun map, void *data)
        1333                 : {
        1334               0 :     int ct = 0;
        1335               0 :     for (RootEnum e(rt->gcRootsHash); !e.empty(); e.popFront()) {
        1336               0 :         RootEntry &entry = e.front();
        1337                 :
        1338               0 :         ct++;
        1339               0 :         int mapflags = map(entry.key, entry.value.type, entry.value.name, data);
        1340                 :
        1341               0 :         if (mapflags & JS_MAP_GCROOT_REMOVE)
        1342               0 :             e.removeFront();
        1343               0 :         if (mapflags & JS_MAP_GCROOT_STOP)
        1344               0 :             break;
        1345                 :     }
        1346                 :
        1347               0 :     return ct;
        1348                 : }
        1349                 :
        1350                 : static size_t
        1351          205438 : ComputeTriggerBytes(size_t lastBytes, size_t maxBytes, JSGCInvocationKind gckind)
        1352                 : {
        1353          205438 :     size_t base = gckind == GC_SHRINK ? lastBytes : Max(lastBytes, GC_ALLOCATION_THRESHOLD);
        1354          205438 :     float trigger = float(base) * GC_HEAP_GROWTH_FACTOR;
        1355          205438 :     return size_t(Min(float(maxBytes), trigger));
        1356                 : }
        1357                 :
        1358                 : void
        1359          102719 : JSCompartment::setGCLastBytes(size_t lastBytes, size_t lastMallocBytes, JSGCInvocationKind gckind)
        1360                 : {
        1361          102719 :     gcTriggerBytes = ComputeTriggerBytes(lastBytes, rt->gcMaxBytes, gckind);
        1362          102719 :     gcTriggerMallocAndFreeBytes = ComputeTriggerBytes(lastMallocBytes, SIZE_MAX, gckind);
        1363          102719 : }
        1364                 :
        1365                 : void
        1366          220528 : JSCompartment::reduceGCTriggerBytes(size_t amount)
        1367                 : {
        1368          220528 :     JS_ASSERT(amount > 0);
        1369          220528 :     JS_ASSERT(gcTriggerBytes >= amount);
        1370          220528 :     if (gcTriggerBytes - amount < GC_ALLOCATION_THRESHOLD * GC_HEAP_GROWTH_FACTOR)
        1371           89538 :         return;
        1372          130990 :     gcTriggerBytes -= amount;
        1373                 : }
        1374                 :
        1375                 : namespace js {
        1376                 : namespace gc {
        1377                 :
        1378                 : inline void
        1379            5399 : ArenaLists::prepareForIncrementalGC(JSRuntime *rt)
        1380                 : {
        1381          113379 :     for (size_t i = 0; i != FINALIZE_LIMIT; ++i) {
        1382          107980 :         FreeSpan *headSpan = &freeLists[i];
        1383          107980 :         if (!headSpan->isEmpty()) {
        1384           51083 :             ArenaHeader *aheader = headSpan->arenaHeader();
        1385           51083 :             aheader->allocatedDuringIncremental = true;
        1386           51083 :             rt->gcMarker.delayMarkingArena(aheader);
        1387                 :         }
        1388                 :     }
        1389            5399 : }
        1390                 :
        1391                 : inline void *
        1392         1768506 : ArenaLists::allocateFromArena(JSCompartment *comp, AllocKind thingKind)
        1393                 : {
        1394         1768506 :     Chunk *chunk = NULL;
        1395                 :
        1396         1768506 :     ArenaList *al = &arenaLists[thingKind];
        1397         3537012 :     AutoLockGC maybeLock;
        1398                 :
        1399                 : #ifdef JS_THREADSAFE
        1400         1768506 :     volatile uintptr_t *bfs = &backgroundFinalizeState[thingKind];
        1401         1768506 :     if (*bfs != BFS_DONE) {
        1402                 :         /*
        1403                 :          * We cannot search the arena list for free things while the
        1404                 :          * background finalization runs and can modify head or cursor at any
        1405                 :          * moment. So we always allocate a new arena in that case.
        1406                 :          */
        1407           47211 :         maybeLock.lock(comp->rt);
        1408           47211 :         if (*bfs == BFS_RUN) {
        1409           28488 :             JS_ASSERT(!*al->cursor);
        1410           28488 :             chunk = PickChunk(comp);
        1411           28488 :             if (!chunk) {
        1412                 :                 /*
        1413                 :                  * Let the caller to wait for the background allocation to
        1414                 :                  * finish and restart the allocation attempt.
        1415                 :                  */
        1416               0 :                 return NULL;
        1417                 :             }
        1418           18723 :         } else if (*bfs == BFS_JUST_FINISHED) {
        1419                 :             /* See comments before BackgroundFinalizeState definition. */
        1420           18723 :             *bfs = BFS_DONE;
        1421                 :         } else {
        1422               0 :             JS_ASSERT(*bfs == BFS_DONE);
        1423                 :         }
        1424                 :     }
        1425                 : #endif /* JS_THREADSAFE */
        1426                 :
        1427         1768506 :     if (!chunk) {
        1428         1740018 :         if (ArenaHeader *aheader = *al->cursor) {
        1429           85138 :             JS_ASSERT(aheader->hasFreeThings());
        1430                 :
        1431                 :             /*
        1432                 :              * The empty arenas are returned to the chunk and should not present on
        1433                 :              * the list.
        1434                 :              */
        1435           85138 :             JS_ASSERT(!aheader->isEmpty());
        1436           85138 :             al->cursor = &aheader->next;
        1437                 :
        1438                 :             /*
        1439                 :              * Move the free span stored in the arena to the free list and
        1440                 :              * allocate from it.
        1441                 :              */
        1442           85138 :             freeLists[thingKind] = aheader->getFirstFreeSpan();
        1443           85138 :             aheader->setAsFullyUsed();
        1444           85138 :             if (JS_UNLIKELY(comp->needsBarrier())) {
        1445              75 :                 aheader->allocatedDuringIncremental = true;
        1446              75 :                 comp->rt->gcMarker.delayMarkingArena(aheader);
        1447                 :             }
        1448           85138 :             return freeLists[thingKind].infallibleAllocate(Arena::thingSize(thingKind));
        1449                 :         }
        1450                 :
        1451                 :         /* Make sure we hold the GC lock before we call PickChunk. */
        1452         1654880 :         if (!maybeLock.locked())
        1453         1654870 :             maybeLock.lock(comp->rt);
        1454         1654880 :         chunk = PickChunk(comp);
        1455         1654880 :         if (!chunk)
        1456               0 :             return NULL;
        1457                 :     }
        1458                 :
        1459                 :     /*
        1460                 :      * While we still hold the GC lock get an arena from some chunk, mark it
        1461                 :      * as full as its single free span is moved to the free lits, and insert
        1462                 :      * it to the list as a fully allocated arena.
        1463                 :      *
        1464                 :      * We add the arena before the the head, not after the tail pointed by the
        1465                 :      * cursor, so after the GC the most recently added arena will be used first
        1466                 :      * for allocations improving cache locality.
        1467                 :      */
        1468         1683368 :     JS_ASSERT(!*al->cursor);
        1469         1683368 :     ArenaHeader *aheader = chunk->allocateArena(comp, thingKind);
        1470         1683368 :     if (!aheader)
        1471              41 :         return NULL;
        1472                 :
        1473         1683327 :     if (JS_UNLIKELY(comp->needsBarrier())) {
        1474              63 :         aheader->allocatedDuringIncremental = true;
        1475              63 :         comp->rt->gcMarker.delayMarkingArena(aheader);
        1476                 :     }
        1477         1683327 :     aheader->next = al->head;
        1478         1683327 :     if (!al->head) {
        1479          348751 :         JS_ASSERT(al->cursor == &al->head);
        1480          348751 :         al->cursor = &aheader->next;
        1481                 :     }
        1482         1683327 :     al->head = aheader;
        1483                 :
        1484                 :     /* See comments before allocateFromNewArena about this assert. */
        1485         1683327 :     JS_ASSERT(!aheader->hasFreeThings());
        1486         1683327 :     uintptr_t arenaAddr = aheader->arenaAddress();
        1487         1683327 :     return freeLists[thingKind].allocateFromNewArena(arenaAddr,
        1488                 :                                                      Arena::firstThingOffset(thingKind),
        1489         3366654 :                                                      Arena::thingSize(thingKind));
        1490                 : }
        1491                 :
        1492                 : void
        1493         1004364 : ArenaLists::finalizeNow(FreeOp *fop, AllocKind thingKind)
        1494                 : {
        1495         1004364 :     JS_ASSERT(!fop->onBackgroundThread());
        1496                 : #ifdef JS_THREADSAFE
        1497         1004364 :     JS_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
        1498                 : #endif
        1499         1004364 :     FinalizeArenas(fop, &arenaLists[thingKind], thingKind);
        1500         1004364 : }
        1501                 :
        1502                 : inline void
        1503          669576 : ArenaLists::finalizeLater(FreeOp *fop, AllocKind thingKind)
        1504                 : {
        1505               0 :     JS_ASSERT(thingKind == FINALIZE_OBJECT0_BACKGROUND  ||
        1506                 :               thingKind == FINALIZE_OBJECT2_BACKGROUND  ||
        1507                 :               thingKind == FINALIZE_OBJECT4_BACKGROUND  ||
        1508                 :               thingKind == FINALIZE_OBJECT8_BACKGROUND  ||
        1509                 :               thingKind == FINALIZE_OBJECT12_BACKGROUND ||
        1510                 :               thingKind == FINALIZE_OBJECT16_BACKGROUND ||
        1511                 :               thingKind == FINALIZE_SHORT_STRING        ||
        1512          669576 :               thingKind == FINALIZE_STRING);
        1513          669576 :     JS_ASSERT(!fop->onBackgroundThread());
        1514                 :
        1515                 : #ifdef JS_THREADSAFE
        1516          669576 :     JS_ASSERT(!fop->runtime()->gcHelperThread.sweeping());
        1517                 :
        1518          669576 :     ArenaList *al = &arenaLists[thingKind];
        1519          669576 :     if (!al->head) {
        1520          355616 :         JS_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE);
        1521          355616 :         JS_ASSERT(al->cursor == &al->head);
        1522          355616 :         return;
        1523                 :     }
        1524                 :
        1525                 :     /*
        1526                 :      * The state can be just-finished if we have not allocated any GC things
        1527                 :      * from the arena list after the previous background finalization.
        1528                 :      */
        1529          450444 :     JS_ASSERT(backgroundFinalizeState[thingKind] == BFS_DONE ||
        1530          764404 :               backgroundFinalizeState[thingKind] == BFS_JUST_FINISHED);
        1531                 :
        1532          313960 :     if (fop->shouldFreeLater()) {
        1533                 :         /*
        1534                 :          * To ensure the finalization order even during the background GC we
        1535                 :          * must use infallibleAppend so arenas scheduled for background
        1536                 :          * finalization would not be finalized now if the append fails.
        1537                 :          */
        1538          156948 :         fop->runtime()->gcHelperThread.finalizeVector.infallibleAppend(al->head);
        1539          156948 :         al->clear();
        1540          156948 :         backgroundFinalizeState[thingKind] = BFS_RUN;
        1541                 :     } else {
        1542          157012 :         FinalizeArenas(fop, al, thingKind);
        1543          157012 :         backgroundFinalizeState[thingKind] = BFS_DONE;
        1544                 :     }
        1545                 :
        1546                 : #else /* !JS_THREADSAFE */
        1547                 :
        1548                 :     finalizeNow(fop, thingKind);
        1549                 :
        1550                 : #endif
        1551                 : }
        1552                 :
        1553                 : #ifdef JS_THREADSAFE
        1554                 : /*static*/ void
        1555          156948 : ArenaLists::backgroundFinalize(FreeOp *fop, ArenaHeader *listHead)
        1556                 : {
        1557          156948 :     JS_ASSERT(fop->onBackgroundThread());
        1558          156948 :     JS_ASSERT(listHead);
        1559          156948 :     AllocKind thingKind = listHead->getAllocKind();
        1560          156948 :     JSCompartment *comp = listHead->compartment;
        1561          156948 :     ArenaList finalized;
        1562          156948 :     finalized.head = listHead;
        1563          156948 :     FinalizeArenas(fop, &finalized, thingKind);
        1564                 :
        1565                 :     /*
        1566                 :      * After we finish the finalization al->cursor must point to the end of
        1567                 :      * the head list as we emptied the list before the background finalization
        1568                 :      * and the allocation adds new arenas before the cursor.
        1569                 :      */
        1570          156948 :     ArenaLists *lists = &comp->arenas;
        1571          156948 :     ArenaList *al = &lists->arenaLists[thingKind];
        1572                 :
        1573          313896 :     AutoLockGC lock(fop->runtime());
        1574          156948 :     JS_ASSERT(lists->backgroundFinalizeState[thingKind] == BFS_RUN);
        1575          156948 :     JS_ASSERT(!*al->cursor);
        1576                 :
        1577                 :     /*
        1578                 :      * We must set the state to BFS_JUST_FINISHED if we touch arenaList list,
        1579                 :      * even if we add to the list only fully allocated arenas without any free
        1580                 :      * things. It ensures that the allocation thread takes the GC lock and all
        1581                 :      * writes to the free list elements are propagated. As we always take the
        1582                 :      * GC lock when allocating new arenas from the chunks we can set the state
        1583                 :      * to BFS_DONE if we have released all finalized arenas back to their
        1584                 :      * chunks.
        1585                 :      */
        1586          156948 :     if (finalized.head) {
        1587          155207 :         *al->cursor = finalized.head;
        1588          155207 :         if (finalized.cursor != &finalized.head)
        1589           59718 :             al->cursor = finalized.cursor;
        1590          155207 :         lists->backgroundFinalizeState[thingKind] = BFS_JUST_FINISHED;
        1591                 :     } else {
        1592            1741 :         lists->backgroundFinalizeState[thingKind] = BFS_DONE;
        1593                 :     }
        1594          156948 : }
        1595                 : #endif /* JS_THREADSAFE */
        1596                 :
        1597                 : void
        1598           83697 : ArenaLists::finalizeObjects(FreeOp *fop)
        1599                 : {
        1600           83697 :     finalizeNow(fop, FINALIZE_OBJECT0);
        1601           83697 :     finalizeNow(fop, FINALIZE_OBJECT2);
        1602           83697 :     finalizeNow(fop, FINALIZE_OBJECT4);
        1603           83697 :     finalizeNow(fop, FINALIZE_OBJECT8);
        1604           83697 :     finalizeNow(fop, FINALIZE_OBJECT12);
        1605           83697 :     finalizeNow(fop, FINALIZE_OBJECT16);
        1606                 :
        1607                 : #ifdef JS_THREADSAFE
        1608           83697 :     finalizeLater(fop, FINALIZE_OBJECT0_BACKGROUND);
        1609           83697 :     finalizeLater(fop, FINALIZE_OBJECT2_BACKGROUND);
        1610           83697 :     finalizeLater(fop, FINALIZE_OBJECT4_BACKGROUND);
        1611           83697 :     finalizeLater(fop, FINALIZE_OBJECT8_BACKGROUND);
        1612           83697 :     finalizeLater(fop, FINALIZE_OBJECT12_BACKGROUND);
        1613           83697 :     finalizeLater(fop, FINALIZE_OBJECT16_BACKGROUND);
        1614                 : #endif
        1615                 :
        1616                 : #if JS_HAS_XML_SUPPORT
        1617           83697 :     finalizeNow(fop, FINALIZE_XML);
        1618                 : #endif
        1619           83697 : }
        1620                 :
        1621                 : void
        1622           83697 : ArenaLists::finalizeStrings(FreeOp *fop)
        1623                 : {
        1624           83697 :     finalizeLater(fop, FINALIZE_SHORT_STRING);
        1625           83697 :     finalizeLater(fop, FINALIZE_STRING);
        1626                 :
        1627           83697 :     finalizeNow(fop, FINALIZE_EXTERNAL_STRING);
        1628           83697 : }
        1629                 :
        1630                 : void
        1631           83697 : ArenaLists::finalizeShapes(FreeOp *fop)
        1632                 : {
        1633           83697 :     finalizeNow(fop, FINALIZE_SHAPE);
        1634           83697 :     finalizeNow(fop, FINALIZE_BASE_SHAPE);
        1635           83697 :     finalizeNow(fop, FINALIZE_TYPE_OBJECT);
        1636           83697 : }
        1637                 :
        1638                 : void
        1639           83697 : ArenaLists::finalizeScripts(FreeOp *fop)
        1640                 : {
        1641           83697 :     finalizeNow(fop, FINALIZE_SCRIPT);
        1642           83697 : }
        1643                 :
        1644                 : static void
        1645           10674 : RunLastDitchGC(JSContext *cx, gcreason::Reason reason)
        1646                 : {
        1647           10674 :     JSRuntime *rt = cx->runtime;
        1648                 :
        1649                 :     /* The last ditch GC preserves all atoms. */
        1650           21348 :     AutoKeepAtoms keep(rt);
        1651           10674 :     GC(cx, GC_NORMAL, reason);
        1652           10674 : }
        1653                 :
        1654                 : /* static */ void *
        1655         1768475 : ArenaLists::refillFreeList(JSContext *cx, AllocKind thingKind)
        1656                 : {
        1657         1768475 :     JS_ASSERT(cx->compartment->arenas.freeLists[thingKind].isEmpty());
        1658                 :
        1659         1768475 :     JSCompartment *comp = cx->compartment;
        1660         1768475 :     JSRuntime *rt = comp->rt;
        1661         1768475 :     JS_ASSERT(!rt->gcRunning);
        1662                 :
        1663         1768475 :     bool runGC = rt->gcIncrementalState != NO_INCREMENTAL && comp->gcBytes > comp->gcTriggerBytes;
        1664              10 :     for (;;) {
        1665         1768485 :         if (JS_UNLIKELY(runGC)) {
        1666              10 :             PrepareCompartmentForGC(comp);
        1667              10 :             RunLastDitchGC(cx, gcreason::LAST_DITCH);
        1668                 :
        1669                 :             /*
        1670                 :              * The JSGC_END callback can legitimately allocate new GC
        1671                 :              * things and populate the free list. If that happens, just
        1672                 :              * return that list head.
        1673                 :              */
        1674              10 :             size_t thingSize = Arena::thingSize(thingKind);
        1675              10 :             if (void *thing = comp->arenas.allocateFromFreeList(thingKind, thingSize))
        1676               0 :                 return thing;
        1677                 :         }
        1678                 :
        1679                 :         /*
        1680                 :          * allocateFromArena may fail while the background finalization still
        1681                 :          * run. In that case we want to wait for it to finish and restart.
        1682                 :          * However, checking for that is racy as the background finalization
        1683                 :          * could free some things after allocateFromArena decided to fail but
        1684                 :          * at this point it may have already stopped. To avoid this race we
        1685                 :          * always try to allocate twice.
        1686                 :          */
        1687         1768506 :         for (bool secondAttempt = false; ; secondAttempt = true) {
        1688         1768506 :             void *thing = comp->arenas.allocateFromArena(comp, thingKind);
        1689         1768506 :             if (JS_LIKELY(!!thing))
        1690         1768465 :                 return thing;
        1691              41 :             if (secondAttempt)
        1692                 :                 break;
        1693                 :
        1694              42 :             AutoLockGC lock(rt);
        1695                 : #ifdef JS_THREADSAFE
        1696              21 :             rt->gcHelperThread.waitBackgroundSweepEnd();
        1697                 : #endif
        1698                 :         }
        1699                 :
        1700                 :         /*
        1701                 :          * We failed to allocate. Run the GC if we haven't done it already.
        1702                 :          * Otherwise report OOM.
        1703                 :          */
        1704              20 :         if (runGC)
        1705                 :             break;
        1706              10 :         runGC = true;
        1707                 :     }
        1708                 :
        1709              10 :     js_ReportOutOfMemory(cx);
        1710              10 :     return NULL;
        1711                 : }
        1712                 :
        1713                 : } /* namespace gc */
        1714                 : } /* namespace js */
        1715                 :
        1716                 : JSGCTraceKind
        1717               0 : js_GetGCThingTraceKind(void *thing)
        1718                 : {
        1719               0 :     return GetGCThingTraceKind(thing);
        1720                 : }
        1721                 :
        1722                 : JSBool
        1723               0 : js_LockGCThingRT(JSRuntime *rt, void *thing)
        1724                 : {
        1725               0 :     if (!thing)
        1726               0 :         return true;
        1727                 :
        1728               0 :     if (GCLocks::Ptr p = rt->gcLocksHash.lookupWithDefault(thing, 0)) {
        1729               0 :         p->value++;
        1730               0 :         return true;
        1731                 :     }
        1732                 :
        1733               0 :     return false;
        1734                 : }
        1735                 :
        1736                 : void
        1737               0 : js_UnlockGCThingRT(JSRuntime *rt, void *thing)
        1738                 : {
        1739               0 :     if (!thing)
        1740               0 :         return;
        1741                 :
        1742               0 :     if (GCLocks::Ptr p = rt->gcLocksHash.lookup(thing)) {
        1743               0 :         rt->gcPoke = true;
        1744               0 :         if (--p->value == 0)
        1745               0 :             rt->gcLocksHash.remove(p);
        1746                 :     }
        1747                 : }
        1748                 :
        1749                 : namespace js {
        1750                 :
        1751                 : void
        1752           46061 : InitTracer(JSTracer *trc, JSRuntime *rt, JSTraceCallback callback)
        1753                 : {
        1754           46061 :     trc->runtime = rt;
        1755           46061 :     trc->callback = callback;
        1756           46061 :     trc->debugPrinter = NULL;
        1757           46061 :     trc->debugPrintArg = NULL;
        1758           46061 :     trc->debugPrintIndex = size_t(-1);
        1759           46061 :     trc->eagerlyTraceWeakMaps = true;
        1760           46061 : }
        1761                 :
        1762                 : /* static */ int64_t
        1763               0 : SliceBudget::TimeBudget(int64_t millis)
        1764                 : {
        1765               0 :     return millis * PRMJ_USEC_PER_MSEC;
        1766                 : }
        1767                 :
        1768                 : /* static */ int64_t
        1769              54 : SliceBudget::WorkBudget(int64_t work)
        1770                 : {
        1771                 :     /* For work = 0 not to mean Unlimited, we subtract 1. */
        1772              54 :     return -work - 1;
        1773                 : }
        1774                 :
        1775           77183 : SliceBudget::SliceBudget()
        1776                 :   : deadline(INT64_MAX),
        1777           77183 :     counter(INTPTR_MAX)
        1778                 : {
        1779           77183 : }
        1780                 :
        1781              63 : SliceBudget::SliceBudget(int64_t budget)
        1782                 : {
        1783              63 :     if (budget == Unlimited) {
        1784               9 :         deadline = INT64_MAX;
        1785               9 :         counter = INTPTR_MAX;
        1786              54 :     } else if (budget > 0) {
        1787               0 :         deadline = PRMJ_Now() + budget;
        1788               0 :         counter = CounterReset;
        1789                 :     } else {
        1790              54 :         deadline = 0;
        1791              54 :         counter = -budget - 1;
        1792                 :     }
        1793              63 : }
        1794                 :
        1795                 : bool
        1796              99 : SliceBudget::checkOverBudget()
        1797                 : {
        1798              99 :     bool over = PRMJ_Now() > deadline;
        1799              99 :     if (!over)
        1800               0 :         counter = CounterReset;
        1801              99 :     return over;
        1802                 : }
        1803                 :
        1804           18761 : GCMarker::GCMarker()
        1805                 :   : stack(size_t(-1)),
        1806                 :     color(BLACK),
        1807                 :     started(false),
        1808                 :     unmarkedArenaStackTop(NULL),
        1809                 :     markLaterArenas(0),
        1810           18761 :     grayFailed(false)
        1811                 : {
        1812           18761 : }
        1813                 :
        1814                 : bool
        1815           18761 : GCMarker::init()
        1816                 : {
        1817           18761 :     return stack.init(MARK_STACK_LENGTH);
        1818                 : }
        1819                 :
        1820                 : void
        1821           39964 : GCMarker::start(JSRuntime *rt)
        1822                 : {
        1823           39964 :     InitTracer(this, rt, NULL);
        1824           39964 :     JS_ASSERT(!started);
        1825           39964 :     started = true;
        1826           39964 :     color = BLACK;
        1827                 :
        1828           39964 :     JS_ASSERT(!unmarkedArenaStackTop);
        1829           39964 :     JS_ASSERT(markLaterArenas == 0);
        1830                 :
        1831           39964 :     JS_ASSERT(grayRoots.empty());
        1832           39964 :     JS_ASSERT(!grayFailed);
        1833                 :
        1834                 :     /*
        1835                 :      * The GC is recomputing the liveness of WeakMap entries, so we delay
        1836                 :      * visting entries.
        1837                 :      */
        1838           39964 :     eagerlyTraceWeakMaps = JS_FALSE;
        1839           39964 : }
        1840                 :
        1841                 : void
        1842           39864 : GCMarker::stop()
        1843                 : {
        1844           39864 :     JS_ASSERT(isDrained());
        1845                 :
        1846           39864 :     JS_ASSERT(started);
        1847           39864 :     started = false;
        1848                 :
        1849           39864 :     JS_ASSERT(!unmarkedArenaStackTop);
        1850           39864 :     JS_ASSERT(markLaterArenas == 0);
        1851                 :
        1852           39864 :     JS_ASSERT(grayRoots.empty());
        1853           39864 :     grayFailed = false;
        1854                 :
        1855                 :     /* Free non-ballast stack memory. */
        1856           39864 :     stack.reset();
        1857           39864 :     grayRoots.clearAndFree();
        1858           39864 : }
        1859                 :
        1860                 : void
        1861            1446 : GCMarker::reset()
        1862                 : {
        1863            1446 :     color = BLACK;
        1864                 :
        1865            1446 :     stack.reset();
        1866            1446 :     JS_ASSERT(isMarkStackEmpty());
        1867                 :
        1868           53879 :     while (unmarkedArenaStackTop) {
        1869           50987 :         ArenaHeader *aheader = unmarkedArenaStackTop;
        1870           50987 :         JS_ASSERT(aheader->hasDelayedMarking);
        1871           50987 :         JS_ASSERT(markLaterArenas);
        1872           50987 :         unmarkedArenaStackTop = aheader->getNextDelayedMarking();
        1873           50987 :         aheader->hasDelayedMarking = 0;
        1874           50987 :         aheader->markOverflow = 0;
        1875           50987 :         aheader->allocatedDuringIncremental = 0;
        1876           50987 :         markLaterArenas--;
        1877                 :     }
        1878            1446 :     JS_ASSERT(isDrained());
        1879            1446 :     JS_ASSERT(!markLaterArenas);
        1880                 :
        1881            1446 :     grayRoots.clearAndFree();
        1882            1446 :     grayFailed = false;
        1883            1446 : }
        1884                 :
        1885                 : /*
        1886                 :  * When the native stack is low, the GC does not call JS_TraceChildren to mark
        1887                 :  * the reachable "children" of the thing. Rather the thing is put aside and
        1888                 :  * JS_TraceChildren is called later with more space on the C stack.
        1889                 :  *
        1890                 :  * To implement such delayed marking of the children with minimal overhead for
        1891                 :  * the normal case of sufficient native stack, the code adds a field per
        1892                 :  * arena. The field markingDelay->link links all arenas with delayed things
        1893                 :  * into a stack list with the pointer to stack top in
        1894                 :  * GCMarker::unmarkedArenaStackTop. delayMarkingChildren adds
        1895                 :  * arenas to the stack as necessary while markDelayedChildren pops the arenas
        1896                 :  * from the stack until it empties.
        1897                 :  */
        1898                 :
        1899                 : inline void
        1900           51221 : GCMarker::delayMarkingArena(ArenaHeader *aheader)
        1901                 : {
        1902           51221 :     if (aheader->hasDelayedMarking) {
        1903                 :         /* Arena already scheduled to be marked later */
        1904             225 :         return;
        1905                 :     }
        1906           50996 :     aheader->setNextDelayedMarking(unmarkedArenaStackTop);
        1907           50996 :     unmarkedArenaStackTop = aheader;
        1908           50996 :     markLaterArenas++;
        1909                 : }
        1910                 :
        1911                 : void
        1912               0 : GCMarker::delayMarkingChildren(const void *thing)
        1913                 : {
        1914               0 :     const Cell *cell = reinterpret_cast<const Cell *>(thing);
        1915               0 :     cell->arenaHeader()->markOverflow = 1;
        1916               0 :     delayMarkingArena(cell->arenaHeader());
        1917               0 : }
        1918                 :
        1919                 : void
        1920               9 : GCMarker::markDelayedChildren(ArenaHeader *aheader)
        1921                 : {
        1922               9 :     if (aheader->markOverflow) {
        1923               0 :         bool always = aheader->allocatedDuringIncremental;
        1924               0 :         aheader->markOverflow = 0;
        1925                 :
        1926               0 :         for (CellIterUnderGC i(aheader); !i.done(); i.next()) {
        1927               0 :             Cell *t = i.getCell();
        1928               0 :             if (always || t->isMarked()) {
        1929               0 :                 t->markIfUnmarked();
        1930               0 :                 JS_TraceChildren(this, t, MapAllocToTraceKind(aheader->getAllocKind()));
        1931                 :             }
        1932                 :         }
        1933                 :     } else {
        1934               9 :         JS_ASSERT(aheader->allocatedDuringIncremental);
        1935               9 :         PushArena(this, aheader);
        1936                 :     }
        1937               9 :     aheader->allocatedDuringIncremental = 0;
        1938               9 : }
        1939                 :
        1940                 : bool
        1941               9 : GCMarker::markDelayedChildren(SliceBudget &budget)
        1942                 : {
        1943              18 :     gcstats::AutoPhase ap(runtime->gcStats, gcstats::PHASE_MARK_DELAYED);
        1944                 :
        1945               9 :     JS_ASSERT(unmarkedArenaStackTop);
        1946               9 :     do {
        1947                 :         /*
        1948                 :          * If marking gets delayed at the same arena again, we must repeat
        1949                 :          * marking of its things. For that we pop arena from the stack and
        1950                 :          * clear its hasDelayedMarking flag before we begin the marking.
        1951                 :          */
        1952               9 :         ArenaHeader *aheader = unmarkedArenaStackTop;
        1953               9 :         JS_ASSERT(aheader->hasDelayedMarking);
        1954               9 :         JS_ASSERT(markLaterArenas);
        1955               9 :         unmarkedArenaStackTop = aheader->getNextDelayedMarking();
        1956               9 :         aheader->hasDelayedMarking = 0;
        1957               9 :         markLaterArenas--;
        1958               9 :         markDelayedChildren(aheader);
        1959                 :
        1960               9 :         budget.step(150);
        1961               9 :         if (budget.isOverBudget())
        1962               0 :             return false;
        1963                 :     } while (unmarkedArenaStackTop);
        1964               9 :     JS_ASSERT(!markLaterArenas);
        1965                 :
        1966               9 :     return true;
        1967                 : }
        1968                 :
        1969                 : #ifdef DEBUG
        1970                 : void
        1971        22725226 : GCMarker::checkCompartment(void *p)
        1972                 : {
        1973        22725226 :     JS_ASSERT(started);
        1974        22725226 :     JS_ASSERT(static_cast<Cell *>(p)->compartment()->isCollecting());
        1975        22725226 : }
        1976                 : #endif
        1977                 :
        1978                 : bool
        1979           38499 : GCMarker::hasBufferedGrayRoots() const
        1980                 : {
        1981           38499 :     return !grayFailed;
        1982                 : }
        1983                 :
        1984                 : void
        1985               0 : GCMarker::startBufferingGrayRoots()
        1986                 : {
        1987               0 :     JS_ASSERT(!callback);
        1988               0 :     callback = GrayCallback;
        1989               0 :     JS_ASSERT(IS_GC_MARKING_TRACER(this));
        1990               0 : }
        1991                 :
        1992                 : void
        1993               0 : GCMarker::endBufferingGrayRoots()
        1994                 : {
        1995               0 :     JS_ASSERT(callback == GrayCallback);
        1996               0 :     callback = NULL;
        1997               0 :     JS_ASSERT(IS_GC_MARKING_TRACER(this));
        1998               0 : }
        1999                 :
        2000                 : void
        2001           38436 : GCMarker::markBufferedGrayRoots()
        2002                 : {
        2003           38436 :     JS_ASSERT(!grayFailed);
        2004                 :
        2005           38436 :     for (GrayRoot *elem = grayRoots.begin(); elem != grayRoots.end(); elem++) {
        2006                 : #ifdef DEBUG
        2007               0 :         debugPrinter = elem->debugPrinter;
        2008               0 :         debugPrintArg = elem->debugPrintArg;
        2009               0 :         debugPrintIndex = elem->debugPrintIndex;
        2010                 : #endif
        2011               0 :         void *tmp = elem->thing;
        2012               0 :         MarkKind(this, &tmp, elem->kind);
        2013               0 :         JS_ASSERT(tmp == elem->thing);
        2014                 :     }
        2015                 :
        2016           38436 :     grayRoots.clearAndFree();
        2017           38436 : }
        2018                 :
        2019                 : void
        2020               0 : GCMarker::appendGrayRoot(void *thing, JSGCTraceKind kind)
        2021                 : {
        2022               0 :     JS_ASSERT(started);
        2023                 :
        2024               0 :     if (grayFailed)
        2025               0 :         return;
        2026                 :
        2027               0 :     GrayRoot root(thing, kind);
        2028                 : #ifdef DEBUG
        2029               0 :     root.debugPrinter = debugPrinter;
        2030               0 :     root.debugPrintArg = debugPrintArg;
        2031               0 :     root.debugPrintIndex = debugPrintIndex;
        2032                 : #endif
        2033                 :
        2034               0 :     if (!grayRoots.append(root)) {
        2035               0 :         grayRoots.clearAndFree();
        2036               0 :         grayFailed = true;
        2037                 :     }
        2038                 : }
        2039                 :
        2040                 : void
        2041               0 : GCMarker::GrayCallback(JSTracer *trc, void **thingp, JSGCTraceKind kind)
        2042                 : {
        2043               0 :     GCMarker *gcmarker = static_cast<GCMarker *>(trc);
        2044               0 :     gcmarker->appendGrayRoot(*thingp, kind);
        2045               0 : }
        2046                 :
        2047                 : size_t
        2048               0 : GCMarker::sizeOfExcludingThis(JSMallocSizeOfFun mallocSizeOf) const
        2049                 : {
        2050               0 :     return stack.sizeOfExcludingThis(mallocSizeOf) +
        2051               0 :            grayRoots.sizeOfExcludingThis(mallocSizeOf);
        2052                 : }
        2053                 :
        2054                 : void
        2055               0 : SetMarkStackLimit(JSRuntime *rt, size_t limit)
        2056                 : {
        2057               0 :     JS_ASSERT(!rt->gcRunning);
        2058               0 :     rt->gcMarker.setSizeLimit(limit);
        2059               0 : }
        2060                 :
        2061                 : } /* namespace js */
        2062                 :
        2063                 : static void
        2064           22853 : gc_root_traversal(JSTracer *trc, const RootEntry &entry)
        2065                 : {
        2066           22853 :     const char *name = entry.value.name ? entry.value.name : "root";
        2067           22853 :     if (entry.value.type == JS_GC_ROOT_GCTHING_PTR)
        2068           22828 :         MarkGCThingRoot(trc, reinterpret_cast<void **>(entry.key), name);
        2069                 :     else
        2070              25 :         MarkValueRoot(trc, reinterpret_cast<Value *>(entry.key), name);
        2071           22853 : }
        2072                 :
        2073                 : static void
        2074               0 : gc_lock_traversal(const GCLocks::Entry &entry, JSTracer *trc)
        2075                 : {
        2076               0 :     JS_ASSERT(entry.value >= 1);
        2077               0 :     void *tmp = entry.key;
        2078               0 :     MarkGCThingRoot(trc, &tmp, "locked object");
        2079               0 :     JS_ASSERT(tmp == entry.key);
        2080               0 : }
        2081                 :
        2082                 : namespace js {
        2083                 :
        2084                 : void
        2085              63 : MarkCompartmentActive(StackFrame *fp)
        2086                 : {
        2087              63 :     if (fp->isScriptFrame())
        2088              63 :         fp->script()->compartment()->active = true;
        2089              63 : }
        2090                 :
        2091                 : } /* namespace js */
        2092                 :
        2093                 : void
        2094                 : AutoIdArray::trace(JSTracer *trc)
        2095                 : {
        2096                 :     JS_ASSERT(tag == IDARRAY);
        2097                 :     gc::MarkIdRange(trc, idArray->length, idArray->vector, "JSAutoIdArray.idArray");
        2098                 : }
        2099                 :
        2100                 : void
        2101               0 : AutoEnumStateRooter::trace(JSTracer *trc)
        2102                 : {
        2103               0 :     gc::MarkObjectRoot(trc, &obj, "JS::AutoEnumStateRooter.obj");
        2104               0 : }
        2105                 :
        2106                 : inline void
        2107           19766 : AutoGCRooter::trace(JSTracer *trc)
        2108                 : {
        2109           19766 :     switch (tag) {
        2110                 :       case JSVAL:
        2111            3300 :         MarkValueRoot(trc, &static_cast<AutoValueRooter *>(this)->val, "JS::AutoValueRooter.val");
        2112            3300 :         return;
        2113                 :
        2114                 :       case PARSER:
        2115              15 :         static_cast<Parser *>(this)->trace(trc);
        2116              15 :         return;
        2117                 :
        2118                 :       case ENUMERATOR:
        2119               0 :         static_cast<AutoEnumStateRooter *>(this)->trace(trc);
        2120               0 :         return;
        2121                 :
        2122                 :       case IDARRAY: {
        2123               0 :         JSIdArray *ida = static_cast<AutoIdArray *>(this)->idArray;
        2124               0 :         MarkIdRange(trc, ida->length, ida->vector, "JS::AutoIdArray.idArray");
        2125               0 :         return;
        2126                 :       }
        2127                 :
        2128                 :       case DESCRIPTORS: {
        2129                 :         PropDescArray &descriptors =
        2130               9 :             static_cast<AutoPropDescArrayRooter *>(this)->descriptors;
        2131              18 :         for (size_t i = 0, len = descriptors.length(); i < len; i++) {
        2132               9 :             PropDesc &desc = descriptors[i];
        2133               9 :             MarkValueRoot(trc, &desc.pd, "PropDesc::pd");
        2134               9 :             MarkValueRoot(trc, &desc.value, "PropDesc::value");
        2135               9 :             MarkValueRoot(trc, &desc.get, "PropDesc::get");
        2136               9 :             MarkValueRoot(trc, &desc.set, "PropDesc::set");
        2137                 :         }
        2138               9 :         return;
        2139                 :       }
        2140                 :
        2141                 :       case DESCRIPTOR : {
        2142               9 :         PropertyDescriptor &desc = *static_cast<AutoPropertyDescriptorRooter *>(this);
        2143               9 :         if (desc.obj)
        2144               9 :             MarkObjectRoot(trc, &desc.obj, "Descriptor::obj");
        2145               9 :         MarkValueRoot(trc, &desc.value, "Descriptor::value");
        2146               9 :         if ((desc.attrs & JSPROP_GETTER) && desc.getter) {
        2147               0 :             JSObject *tmp = JS_FUNC_TO_DATA_PTR(JSObject *, desc.getter);
        2148               0 :             MarkObjectRoot(trc, &tmp, "Descriptor::get");
        2149               0 :             desc.getter = JS_DATA_TO_FUNC_PTR(JSPropertyOp, tmp);
        2150                 :         }
        2151               9 :         if (desc.attrs & JSPROP_SETTER && desc.setter) {
        2152               0 :             JSObject *tmp = JS_FUNC_TO_DATA_PTR(JSObject *, desc.setter);
        2153               0 :             MarkObjectRoot(trc, &tmp, "Descriptor::set");
        2154               0 :             desc.setter = JS_DATA_TO_FUNC_PTR(JSStrictPropertyOp, tmp);
        2155                 :         }
        2156               9 :         return;
        2157                 :       }
        2158                 :
        2159                 :       case NAMESPACES: {
        2160               0 :         JSXMLArray<JSObject> &array = static_cast<AutoNamespaceArray *>(this)->array;
        2161               0 :         MarkObjectRange(trc, array.length, array.vector, "JSXMLArray.vector");
        2162               0 :         js_XMLArrayCursorTrace(trc, array.cursors);
        2163               0 :         return;
        2164                 :       }
        2165                 :
        2166                 :       case XML:
        2167               0 :         js_TraceXML(trc, static_cast<AutoXMLRooter *>(this)->xml);
        2168               0 :         return;
        2169                 :
        2170                 :       case OBJECT:
        2171           10031 :         if (static_cast<AutoObjectRooter *>(this)->obj)
        2172                 :             MarkObjectRoot(trc, &static_cast<AutoObjectRooter *>(this)->obj,
        2173           10031 :                            "JS::AutoObjectRooter.obj");
        2174           10031 :         return;
        2175                 :
        2176                 :       case ID:
        2177               0 :         MarkIdRoot(trc, &static_cast<AutoIdRooter *>(this)->id_, "JS::AutoIdRooter.id_");
        2178               0 :         return;
        2179                 :
        2180                 :       case VALVECTOR: {
        2181            2864 :         AutoValueVector::VectorImpl &vector = static_cast<AutoValueVector *>(this)->vector;
        2182            2864 :         MarkValueRootRange(trc, vector.length(), vector.begin(), "js::AutoValueVector.vector");
        2183            2864 :         return;
        2184                 :       }
        2185                 :
        2186                 :       case STRING:
        2187             604 :         if (static_cast<AutoStringRooter *>(this)->str)
        2188                 :             MarkStringRoot(trc, &static_cast<AutoStringRooter *>(this)->str,
        2189             604 :                            "JS::AutoStringRooter.str");
        2190             604 :         return;
        2191                 :
        2192                 :       case IDVECTOR: {
        2193            1242 :         AutoIdVector::VectorImpl &vector = static_cast<AutoIdVector *>(this)->vector;
        2194            1242 :         MarkIdRootRange(trc, vector.length(), vector.begin(), "js::AutoIdVector.vector");
        2195            1242 :         return;
        2196                 :       }
        2197                 :
        2198                 :       case SHAPEVECTOR: {
        2199            1098 :         AutoShapeVector::VectorImpl &vector = static_cast<js::AutoShapeVector *>(this)->vector;
        2200            1098 :         MarkShapeRootRange(trc, vector.length(), const_cast<Shape **>(vector.begin()),
        2201            1098 :                            "js::AutoShapeVector.vector");
        2202            1098 :         return;
        2203                 :       }
        2204                 :
        2205                 :       case OBJVECTOR: {
        2206               9 :         AutoObjectVector::VectorImpl &vector = static_cast<AutoObjectVector *>(this)->vector;
        2207               9 :         MarkObjectRootRange(trc, vector.length(), vector.begin(), "js::AutoObjectVector.vector");
        2208               9 :         return;
        2209                 :       }
        2210                 :
        2211                 :       case VALARRAY: {
        2212               0 :         AutoValueArray *array = static_cast<AutoValueArray *>(this);
        2213               0 :         MarkValueRootRange(trc, array->length(), array->start(), "js::AutoValueArray");
        2214               0 :         return;
        2215                 :       }
        2216                 :
        2217                 :       case SCRIPTVECTOR: {
        2218               0 :         AutoScriptVector::VectorImpl &vector = static_cast<AutoScriptVector *>(this)->vector;
        2219               0 :         for (size_t i = 0; i < vector.length(); i++)
        2220               0 :             MarkScriptRoot(trc, &vector[i], "AutoScriptVector element");
        2221               0 :         return;
        2222                 :       }
        2223                 :     }
        2224                 :
        2225             585 :     JS_ASSERT(tag >= 0);
        2226                 :     MarkValueRootRange(trc, tag, static_cast<AutoArrayRooter *>(this)->array,
        2227             585 :                        "JS::AutoArrayRooter.array");
        2228                 : }
        2229                 :
        2230                 : /* static */ void
        2231           43246 : AutoGCRooter::traceAll(JSTracer *trc)
        2232                 : {
        2233           63012 :     for (js::AutoGCRooter *gcr = trc->runtime->autoGCRooters; gcr; gcr = gcr->down)
        2234           19766 :         gcr->trace(trc);
        2235           43246 : }
        2236                 :
        2237                 : namespace js {
        2238                 :
        2239                 : static void
        2240           41836 : MarkRuntime(JSTracer *trc, bool useSavedRoots = false)
        2241                 : {
        2242           41836 :     JSRuntime *rt = trc->runtime;
        2243           41836 :     JS_ASSERT(trc->callback != GCMarker::GrayCallback);
        2244                 :
        2245           41836 :     if (IS_GC_MARKING_TRACER(trc)) {
        2246          122547 :         for (CompartmentsIter c(rt); !c.done(); c.next()) {
        2247           84084 :             if (!c->isCollecting())
        2248             324 :                 c->markCrossCompartmentWrappers(trc);
        2249                 :         }
        2250           38463 :         Debugger::markCrossCompartmentDebuggerObjectReferents(trc);
        2251                 :     }
        2252                 :
        2253           41836 :     AutoGCRooter::traceAll(trc);
        2254                 :
        2255           41836 :     if (rt->hasContexts())
        2256           22875 :         MarkConservativeStackRoots(trc, useSavedRoots);
        2257                 :
        2258           64689 :     for (RootRange r = rt->gcRootsHash.all(); !r.empty(); r.popFront())
        2259           22853 :         gc_root_traversal(trc, r.front());
        2260                 :
        2261           41836 :     for (GCLocks::Range r = rt->gcLocksHash.all(); !r.empty(); r.popFront())
        2262               0 :         gc_lock_traversal(r.front(), trc);
        2263                 :
        2264           41836 :     if (rt->scriptAndCountsVector) {
        2265               0 :         ScriptAndCountsVector &vec = *rt->scriptAndCountsVector;
        2266               0 :         for (size_t i = 0; i < vec.length(); i++)
        2267               0 :             MarkScriptRoot(trc, &vec[i].script, "scriptAndCountsVector");
        2268                 :     }
        2269                 :
        2270                 :     /*
        2271                 :      * Atoms are not in the cross-compartment map. So if there are any
        2272                 :      * compartments that are not being collected, we are not allowed to collect
        2273                 :      * atoms. Otherwise, the non-collected compartments could contain pointers
        2274                 :      * to atoms that we would miss.
        2275                 :      */
        2276           41836 :     bool isFullGC = true;
        2277           41836 :     if (IS_GC_MARKING_TRACER(trc)) {
        2278          122547 :         for (CompartmentsIter c(rt); !c.done(); c.next()) {
        2279           84084 :             if (!c->isCollecting())
        2280             324 :                 isFullGC = false;
        2281                 :         }
        2282                 :     }
        2283           41836 :     MarkAtomState(trc, rt->gcKeepAtoms || !isFullGC);
        2284           41836 :     rt->staticStrings.trace(trc);
        2285                 :
        2286           64711 :     for (ContextIter acx(rt); !acx.done(); acx.next())
        2287           22875 :         acx->mark(trc);
        2288                 :
        2289                 :     /* We can't use GCCompartmentsIter if we're called from TraceRuntime. */
        2290          137119 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        2291           95283 :         if (IS_GC_MARKING_TRACER(trc) && !c->isCollecting())
        2292             324 :             continue;
        2293                 :
        2294           94959 :         if (c->activeAnalysis)
        2295              20 :             c->markTypes(trc);
        2296                 :
        2297                 :         /* During a GC, these are treated as weak pointers. */
        2298           94959 :         if (!IS_GC_MARKING_TRACER(trc)) {
        2299           11199 :             if (c->watchpointMap)
        2300               0 :                 c->watchpointMap->markAll(trc);
        2301                 :         }
        2302                 :
        2303                 :         /* Do not discard scripts with counts while profiling. */
        2304           94959 :         if (rt->profilingScripts) {
        2305               0 :             for (CellIterUnderGC i(c, FINALIZE_SCRIPT); !i.done(); i.next()) {
        2306               0 :                 JSScript *script = i.get<JSScript>();
        2307               0 :                 if (script->scriptCounts) {
        2308               0 :                     MarkScriptRoot(trc, &script, "profilingScripts");
        2309               0 :                     JS_ASSERT(script == i.get<JSScript>());
        2310                 :                 }
        2311                 :             }
        2312                 :         }
        2313                 :     }
        2314                 :
        2315                 : #ifdef JS_METHODJIT
        2316                 :     /* We need to expand inline frames before stack scanning. */
        2317          137119 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        2318           95283 :         mjit::ExpandInlineFrames(c);
        2319                 : #endif
        2320                 :
        2321           41836 :     rt->stackSpace.mark(trc);
        2322                 :
        2323                 :     /* The embedding can register additional roots here. */
        2324           41836 :     if (JSTraceDataOp op = rt->gcBlackRootsTraceOp)
        2325               0 :         (*op)(trc, rt->gcBlackRootsData);
        2326                 :
        2327                 :     /* During GC, this buffers up the gray roots and doesn't mark them. */
        2328           41836 :     if (JSTraceDataOp op = rt->gcGrayRootsTraceOp) {
        2329               0 :         if (IS_GC_MARKING_TRACER(trc)) {
        2330               0 :             GCMarker *gcmarker = static_cast<GCMarker *>(trc);
        2331               0 :             gcmarker->startBufferingGrayRoots();
        2332               0 :             (*op)(trc, rt->gcGrayRootsData);
        2333               0 :             gcmarker->endBufferingGrayRoots();
        2334                 :         } else {
        2335               0 :             (*op)(trc, rt->gcGrayRootsData);
        2336                 :         }
        2337                 :     }
        2338           41836 : }
        2339                 :
        2340                 : static void
        2341           97833 : TriggerOperationCallback(JSRuntime *rt, gcreason::Reason reason)
        2342                 : {
        2343           97833 :     if (rt->gcIsNeeded)
        2344           97661 :         return;
        2345                 :
        2346             172 :     rt->gcIsNeeded = true;
        2347             172 :     rt->gcTriggerReason = reason;
        2348             172 :     rt->triggerOperationCallback();
        2349                 : }
        2350                 :
        2351                 : void
        2352             165 : TriggerGC(JSRuntime *rt, gcreason::Reason reason)
        2353                 : {
        2354             165 :     JS_ASSERT(rt->onOwnerThread());
        2355                 :
        2356             165 :     if (rt->gcRunning)
        2357               0 :         return;
        2358                 :
        2359             165 :     PrepareForFullGC(rt);
        2360             165 :     TriggerOperationCallback(rt, reason);
        2361                 : }
        2362                 :
        2363                 : void
        2364           97689 : TriggerCompartmentGC(JSCompartment *comp, gcreason::Reason reason)
        2365                 : {
        2366           97689 :     JSRuntime *rt = comp->rt;
        2367           97689 :     JS_ASSERT(rt->onOwnerThread());
        2368                 :
        2369           97689 :     if (rt->gcRunning)
        2370               0 :         return;
        2371                 :
        2372           97689 :     if (rt->gcZeal() == ZealAllocValue) {
        2373               0 :         TriggerGC(rt, reason);
        2374               0 :         return;
        2375                 :     }
        2376                 :
        2377           97689 :     if (comp == rt->atomsCompartment) {
        2378                 :         /* We can't do a compartmental GC of the default compartment. */
        2379              21 :         TriggerGC(rt, reason);
        2380              21 :         return;
        2381                 :     }
        2382                 :
        2383           97668 :     PrepareCompartmentForGC(comp);
        2384           97668 :     TriggerOperationCallback(rt, reason);
        2385                 : }
        2386                 :
        2387                 : void
        2388               0 : MaybeGC(JSContext *cx)
        2389                 : {
        2390               0 :     JSRuntime *rt = cx->runtime;
        2391               0 :     JS_ASSERT(rt->onOwnerThread());
        2392                 :
        2393               0 :     if (rt->gcZeal() == ZealAllocValue || rt->gcZeal() == ZealPokeValue) {
        2394               0 :         PrepareForFullGC(rt);
        2395               0 :         GC(cx, GC_NORMAL, gcreason::MAYBEGC);
        2396               0 :         return;
        2397                 :     }
        2398                 :
        2399               0 :     JSCompartment *comp = cx->compartment;
        2400               0 :     if (rt->gcIsNeeded) {
        2401               0 :         GCSlice(cx, GC_NORMAL, gcreason::MAYBEGC);
        2402               0 :         return;
        2403                 :     }
        2404                 :
        2405               0 :     if (comp->gcBytes > 8192 &&
        2406                 :         comp->gcBytes >= 3 * (comp->gcTriggerBytes / 4) &&
        2407                 :         rt->gcIncrementalState == NO_INCREMENTAL)
        2408                 :     {
        2409               0 :         PrepareCompartmentForGC(comp);
        2410               0 :         GCSlice(cx, GC_NORMAL, gcreason::MAYBEGC);
        2411               0 :         return;
        2412                 :     }
        2413                 :
        2414               0 :     if (comp->gcMallocAndFreeBytes > comp->gcTriggerMallocAndFreeBytes) {
        2415               0 :         PrepareCompartmentForGC(comp);
        2416               0 :         GCSlice(cx, GC_NORMAL, gcreason::MAYBEGC);
        2417               0 :         return;
        2418                 :     }
        2419                 :
        2420                 :     /*
        2421                 :      * Access to the counters and, on 32 bit, setting gcNextFullGCTime below
        2422                 :      * is not atomic and a race condition could trigger or suppress the GC. We
        2423                 :      * tolerate this.
        2424                 :      */
        2425               0 :     int64_t now = PRMJ_Now();
        2426               0 :     if (rt->gcNextFullGCTime && rt->gcNextFullGCTime <= now) {
        2427               0 :         if (rt->gcChunkAllocationSinceLastGC ||
        2428                 :             rt->gcNumArenasFreeCommitted > FreeCommittedArenasThreshold)
        2429                 :         {
        2430               0 :             PrepareForFullGC(rt);
        2431               0 :             GCSlice(cx, GC_SHRINK, gcreason::MAYBEGC);
        2432                 :         } else {
        2433               0 :             rt->gcNextFullGCTime = now + GC_IDLE_FULL_SPAN;
        2434                 :         }
        2435                 :     }
        2436                 : }
        2437                 :
        2438                 : static void
        2439              18 : DecommitArenasFromAvailableList(JSRuntime *rt, Chunk **availableListHeadp)
        2440                 : {
        2441              18 :     Chunk *chunk = *availableListHeadp;
        2442              18 :     if (!chunk)
        2443               0 :         return;
        2444                 :
        2445                 :     /*
        2446                 :      * Decommit is expensive so we avoid holding the GC lock while calling it.
        2447                 :      *
        2448                 :      * We decommit from the tail of the list to minimize interference with the
        2449                 :      * main thread that may start to allocate things at this point.
        2450                 :      *
        2451                 :      * The arena that is been decommitted outside the GC lock must not be
        2452                 :      * available for allocations either via the free list or via the
        2453                 :      * decommittedArenas bitmap. For that we just fetch the arena from the
        2454                 :      * free list before the decommit pretending as it was allocated. If this
        2455                 :      * arena also is the single free arena in the chunk, then we must remove
        2456                 :      * from the available list before we release the lock so the allocation
        2457                 :      * thread would not see chunks with no free arenas on the available list.
        2458                 :      *
        2459                 :      * After we retake the lock, we mark the arena as free and decommitted if
        2460                 :      * the decommit was successful. We must also add the chunk back to the
        2461                 :      * available list if we removed it previously or when the main thread
        2462                 :      * have allocated all remaining free arenas in the chunk.
        2463                 :      *
        2464                 :      * We also must make sure that the aheader is not accessed again after we
        2465                 :      * decommit the arena.
        2466                 :      */
        2467              18 :     JS_ASSERT(chunk->info.prevp == availableListHeadp);
        2468              18 :     while (Chunk *next = chunk->info.next) {
        2469               0 :         JS_ASSERT(next->info.prevp == &chunk->info.next);
        2470               0 :         chunk = next;
        2471                 :     }
        2472                 :
        2473               0 :     for (;;) {
        2474            4167 :         while (chunk->info.numArenasFreeCommitted != 0) {
        2475            4131 :             ArenaHeader *aheader = chunk->fetchNextFreeArena(rt);
        2476                 :
        2477            4131 :             Chunk **savedPrevp = chunk->info.prevp;
        2478            4131 :             if (!chunk->hasAvailableArenas())
        2479               0 :                 chunk->removeFromAvailableList();
        2480                 :
        2481            4131 :             size_t arenaIndex = Chunk::arenaIndex(aheader->arenaAddress());
        2482                 :             bool ok;
        2483                 :             {
        2484                 :                 /*
        2485                 :                  * If the main thread waits for the decommit to finish, skip
        2486                 :                  * potentially expensive unlock/lock pair on the contested
        2487                 :                  * lock.
        2488                 :                  */
        2489            8262 :                 Maybe<AutoUnlockGC> maybeUnlock;
        2490            4131 :                 if (!rt->gcRunning)
        2491            4131 :                     maybeUnlock.construct(rt);
        2492            4131 :                 ok = MarkPagesUnused(aheader->getArena(), ArenaSize);
        2493                 :             }
        2494                 :
        2495            4131 :             if (ok) {
        2496            4131 :                 ++chunk->info.numArenasFree;
        2497            4131 :                 chunk->decommittedArenas.set(arenaIndex);
        2498                 :             } else {
        2499               0 :                 chunk->addArenaToFreeList(rt, aheader);
        2500                 :             }
        2501            4131 :             JS_ASSERT(chunk->hasAvailableArenas());
        2502            4131 :             JS_ASSERT(!chunk->unused());
        2503            4131 :             if (chunk->info.numArenasFree == 1) {
        2504                 :                 /*
        2505                 :                  * Put the chunk back to the available list either at the
        2506                 :                  * point where it was before to preserve the available list
        2507                 :                  * that we enumerate, or, when the allocation thread has fully
        2508                 :                  * used all the previous chunks, at the beginning of the
        2509                 :                  * available list.
        2510                 :                  */
        2511               0 :                 Chunk **insertPoint = savedPrevp;
        2512               0 :                 if (savedPrevp != availableListHeadp) {
        2513               0 :                     Chunk *prev = Chunk::fromPointerToNext(savedPrevp);
        2514               0 :                     if (!prev->hasAvailableArenas())
        2515               0 :                         insertPoint = availableListHeadp;
        2516                 :                 }
        2517               0 :                 chunk->insertToAvailableList(insertPoint);
        2518                 :             } else {
        2519            4131 :                 JS_ASSERT(chunk->info.prevp);
        2520                 :             }
        2521                 :
        2522            4131 :             if (rt->gcChunkAllocationSinceLastGC) {
        2523                 :                 /*
        2524                 :                  * The allocator thread has started to get new chunks. We should stop
        2525                 :                  * to avoid decommitting arenas in just allocated chunks.
        2526                 :                  */
        2527               0 :                 return;
        2528                 :             }
        2529                 :         }
        2530                 :
        2531                 :         /*
        2532                 :          * chunk->info.prevp becomes null when the allocator thread consumed
        2533                 :          * all chunks from the available list.
        2534                 :          */
        2535              18 :         JS_ASSERT_IF(chunk->info.prevp, *chunk->info.prevp == chunk);
        2536              18 :         if (chunk->info.prevp == availableListHeadp || !chunk->info.prevp)
        2537              18 :             break;
        2538                 :
        2539                 :         /*
        2540                 :          * prevp exists and is not the list head. It must point to the next
        2541                 :          * field of the previous chunk.
        2542                 :          */
        2543               0 :         chunk = chunk->getPrevious();
        2544                 :     }
        2545                 : }
        2546                 :
        2547                 : static void
        2548               9 : DecommitArenas(JSRuntime *rt)
        2549                 : {
        2550               9 :     DecommitArenasFromAvailableList(rt, &rt->gcSystemAvailableChunkListHead);
        2551               9 :     DecommitArenasFromAvailableList(rt, &rt->gcUserAvailableChunkListHead);
        2552               9 : }
        2553                 :
        2554                 : /* Must be called with the GC lock taken. */
        2555                 : static void
        2556           19675 : ExpireChunksAndArenas(JSRuntime *rt, bool shouldShrink)
        2557                 : {
        2558           19675 :     if (Chunk *toFree = rt->gcChunkPool.expire(rt, shouldShrink)) {
        2559              18 :         AutoUnlockGC unlock(rt);
        2560               9 :         FreeChunkList(toFree);
        2561                 :     }
        2562                 :
        2563           19675 :     if (shouldShrink)
        2564               9 :         DecommitArenas(rt);
        2565           19675 : }
        2566                 :
        2567                 : #ifdef JS_THREADSAFE
        2568                 :
        2569                 : static unsigned
        2570           18761 : GetCPUCount()
        2571                 : {
        2572                 :     static unsigned ncpus = 0;
        2573           18761 :     if (ncpus == 0) {
        2574                 : # ifdef XP_WIN
        2575                 :         SYSTEM_INFO sysinfo;
        2576                 :         GetSystemInfo(&sysinfo);
        2577                 :         ncpus = unsigned(sysinfo.dwNumberOfProcessors);
        2578                 : # else
        2579           18667 :         long n = sysconf(_SC_NPROCESSORS_ONLN);
        2580           18667 :         ncpus = (n > 0) ? unsigned(n) : 1;
        2581                 : # endif
        2582                 :     }
        2583           18761 :     return ncpus;
        2584                 : }
        2585                 :
        2586                 : bool
        2587           18761 : GCHelperThread::init()
        2588                 : {
        2589           18761 :     if (!(wakeup = PR_NewCondVar(rt->gcLock)))
        2590               0 :         return false;
        2591           18761 :     if (!(done = PR_NewCondVar(rt->gcLock)))
        2592               0 :         return false;
        2593                 :
        2594                 :     thread = PR_CreateThread(PR_USER_THREAD, threadMain, this, PR_PRIORITY_NORMAL,
        2595           18761 :                              PR_LOCAL_THREAD, PR_JOINABLE_THREAD, 0);
        2596           18761 :     if (!thread)
        2597               0 :         return false;
        2598                 :
        2599           18761 :     backgroundAllocation = (GetCPUCount() >= 2);
        2600           18761 :     return true;
        2601                 : }
        2602                 :
        2603                 : void
        2604           18761 : GCHelperThread::finish()
        2605                 : {
        2606           18761 :     PRThread *join = NULL;
        2607                 :     {
        2608           37522 :         AutoLockGC lock(rt);
        2609           18761 :         if (thread && state != SHUTDOWN) {
        2610                 :             /*
        2611                 :              * We cannot be in the ALLOCATING or CANCEL_ALLOCATION states as
        2612                 :              * the allocations should have been stopped during the last GC.
        2613                 :              */
        2614           18761 :             JS_ASSERT(state == IDLE || state == SWEEPING);
        2615           18761 :             if (state == IDLE)
        2616           18761 :                 PR_NotifyCondVar(wakeup);
        2617           18761 :             state = SHUTDOWN;
        2618           18761 :             join = thread;
        2619                 :         }
        2620                 :     }
        2621           18761 :     if (join) {
        2622                 :         /* PR_DestroyThread is not necessary. */
        2623           18761 :         PR_JoinThread(join);
        2624                 :     }
        2625           18761 :     if (wakeup)
        2626           18761 :         PR_DestroyCondVar(wakeup);
        2627           18761 :     if (done)
        2628           18761 :         PR_DestroyCondVar(done);
        2629           18761 : }
        2630                 :
        2631                 : /* static */
        2632                 : void
        2633           18761 : GCHelperThread::threadMain(void *arg)
        2634                 : {
        2635           18761 :     static_cast<GCHelperThread *>(arg)->threadLoop();
        2636           18761 : }
        2637                 :
        2638                 : void
        2639           18761 : GCHelperThread::threadLoop()
        2640                 : {
        2641           37522 :     AutoLockGC lock(rt);
        2642                 :
        2643                 :     /*
        2644                 :      * Even on the first iteration the state can be SHUTDOWN or SWEEPING if
        2645                 :      * the stop request or the GC and the corresponding startBackgroundSweep call
        2646                 :      * happen before this thread has a chance to run.
        2647                 :      */
        2648           63133 :     for (;;) {
        2649           81894 :         switch (state) {
        2650                 :           case SHUTDOWN:
        2651                 :             return;
        2652                 :           case IDLE:
        2653           40947 :             PR_WaitCondVar(wakeup, PR_INTERVAL_NO_TIMEOUT);
        2654           40947 :             break;
        2655                 :           case SWEEPING:
        2656           19675 :             doSweep();
        2657           19675 :             if (state == SWEEPING)
        2658           19675 :                 state = IDLE;
        2659           19675 :             PR_NotifyAllCondVar(done);
        2660           19675 :             break;
        2661                 :           case ALLOCATING:
        2662            5022 :             do {
        2663                 :                 Chunk *chunk;
        2664                 :                 {
        2665            5022 :                     AutoUnlockGC unlock(rt);
        2666            2511 :                     chunk = Chunk::allocate(rt);
        2667                 :                 }
        2668                 :
        2669                 :                 /* OOM stops the background allocation. */
        2670            2511 :                 if (!chunk)
        2671               0 :                     break;
        2672            2511 :                 JS_ASSERT(chunk->info.numArenasFreeCommitted == ArenasPerChunk);
        2673            2511 :                 rt->gcNumArenasFreeCommitted += ArenasPerChunk;
        2674            2511 :                 rt->gcChunkPool.put(chunk);
        2675            2511 :             } while (state == ALLOCATING && rt->gcChunkPool.wantBackgroundAllocation(rt));
        2676            2511 :             if (state == ALLOCATING)
        2677            2511 :                 state = IDLE;
        2678            2511 :             break;
        2679                 :           case CANCEL_ALLOCATION:
        2680               0 :             state = IDLE;
        2681               0 :             PR_NotifyAllCondVar(done);
        2682               0 :             break;
        2683                 :         }
        2684                 :     }
        2685                 : }
        2686                 :
        2687                 : bool
        2688           19666 : GCHelperThread::prepareForBackgroundSweep()
        2689                 : {
        2690           19666 :     JS_ASSERT(state == IDLE);
        2691           19666 :     size_t maxArenaLists = MAX_BACKGROUND_FINALIZE_KINDS * rt->compartments.length();
        2692           19666 :     return finalizeVector.reserve(maxArenaLists);
        2693                 : }
        2694                 :
        2695                 : /* Must be called with the GC lock taken. */
        2696                 : void
        2697           19666 : GCHelperThread::startBackgroundSweep(JSContext *cx, bool shouldShrink)
        2698                 : {
        2699                 :     /* The caller takes the GC lock. */
        2700           19666 :     JS_ASSERT(state == IDLE);
        2701           19666 :     JS_ASSERT(cx);
        2702           19666 :     JS_ASSERT(!finalizationContext);
        2703           19666 :     finalizationContext = cx;
        2704           19666 :     shrinkFlag = shouldShrink;
        2705           19666 :     state = SWEEPING;
        2706           19666 :     PR_NotifyCondVar(wakeup);
        2707           19666 : }
        2708                 :
        2709                 : /* Must be called with the GC lock taken. */
        2710                 : void
        2711               9 : GCHelperThread::startBackgroundShrink()
        2712                 : {
        2713               9 :     switch (state) {
        2714                 :       case IDLE:
        2715               9 :         JS_ASSERT(!finalizationContext);
        2716               9 :         shrinkFlag = true;
        2717               9 :         state = SWEEPING;
        2718               9 :         PR_NotifyCondVar(wakeup);
        2719               9 :         break;
        2720                 :       case SWEEPING:
        2721               0 :         shrinkFlag = true;
        2722               0 :         break;
        2723                 :       case ALLOCATING:
        2724                 :       case CANCEL_ALLOCATION:
        2725                 :         /*
        2726                 :          * If we have started background allocation there is nothing to
        2727                 :          * shrink.
        2728                 :          */
        2729               0 :         break;
        2730                 :       case SHUTDOWN:
        2731               0 :         JS_NOT_REACHED("No shrink on shutdown");
        2732                 :     }
        2733               9 : }
        2734                 :
        2735                 : /* Must be called with the GC lock taken. */
        2736                 : void
        2737           38102 : GCHelperThread::waitBackgroundSweepEnd()
        2738                 : {
        2739           76964 :     while (state == SWEEPING)
        2740             760 :         PR_WaitCondVar(done, PR_INTERVAL_NO_TIMEOUT);
        2741           38102 : }
        2742                 :
        2743                 : /* Must be called with the GC lock taken. */
        2744                 : void
        2745           41410 : GCHelperThread::waitBackgroundSweepOrAllocEnd()
        2746                 : {
        2747           41410 :     if (state == ALLOCATING)
        2748               0 :         state = CANCEL_ALLOCATION;
        2749           90052 :     while (state == SWEEPING || state == CANCEL_ALLOCATION)
        2750            7232 :         PR_WaitCondVar(done, PR_INTERVAL_NO_TIMEOUT);
        2751           41410 : }
        2752                 :
        2753                 : /* Must be called with the GC lock taken. */
        2754                 : inline void
        2755            2511 : GCHelperThread::startBackgroundAllocationIfIdle()
        2756                 : {
        2757            2511 :     if (state == IDLE) {
        2758            2511 :         state = ALLOCATING;
        2759            2511 :         PR_NotifyCondVar(wakeup);
        2760                 :     }
        2761            2511 : }
        2762                 :
        2763                 : JS_FRIEND_API(void)
        2764           12426 : GCHelperThread::replenishAndFreeLater(void *ptr)
        2765                 : {
        2766           12426 :     JS_ASSERT(freeCursor == freeCursorEnd);
        2767                 :     do {
        2768           12426 :         if (freeCursor && !freeVector.append(freeCursorEnd - FREE_ARRAY_LENGTH))
        2769               0 :             break;
        2770           12426 :         freeCursor = (void **) OffTheBooks::malloc_(FREE_ARRAY_SIZE);
        2771           12426 :         if (!freeCursor) {
        2772               0 :             freeCursorEnd = NULL;
        2773               0 :             break;
        2774                 :         }
        2775           12426 :         freeCursorEnd = freeCursor + FREE_ARRAY_LENGTH;
        2776           12426 :         *freeCursor++ = ptr;
        2777           12426 :         return;
        2778                 :     } while (false);
        2779               0 :     Foreground::free_(ptr);
        2780                 : }
        2781                 :
        2782                 : /* Must be called with the GC lock taken. */
        2783                 : void
        2784           19675 : GCHelperThread::doSweep()
        2785                 : {
        2786           19675 :     if (finalizationContext) {
        2787           19666 :         finalizationContext = NULL;
        2788           39332 :         AutoUnlockGC unlock(rt);
        2789                 :
        2790                 :         /*
        2791                 :          * We must finalize in the insert order, see comments in
        2792                 :          * finalizeObjects.
        2793                 :          */
        2794           19666 :         FreeOp fop(rt, false, true);
        2795          176614 :         for (ArenaHeader **i = finalizeVector.begin(); i != finalizeVector.end(); ++i)
        2796          156948 :             ArenaLists::backgroundFinalize(&fop, *i);
        2797           19666 :         finalizeVector.resize(0);
        2798                 :
        2799           19666 :         if (freeCursor) {
        2800           12426 :             void **array = freeCursorEnd - FREE_ARRAY_LENGTH;
        2801           12426 :             freeElementsAndArray(array, freeCursor);
        2802           12426 :             freeCursor = freeCursorEnd = NULL;
        2803                 :         } else {
        2804            7240 :             JS_ASSERT(!freeCursorEnd);
        2805                 :         }
        2806           19666 :         for (void ***iter = freeVector.begin(); iter != freeVector.end(); ++iter) {
        2807               0 :             void **array = *iter;
        2808               0 :             freeElementsAndArray(array, array + FREE_ARRAY_LENGTH);
        2809                 :         }
        2810           19666 :         freeVector.resize(0);
        2811                 :     }
        2812                 :
        2813           19675 :     bool shrinking = shrinkFlag;
        2814           19675 :     ExpireChunksAndArenas(rt, shrinking);
        2815                 :
        2816                 :     /*
        2817                 :      * The main thread may have called ShrinkGCBuffers while
        2818                 :      * ExpireChunksAndArenas(rt, false) was running, so we recheck the flag
        2819                 :      * afterwards.
        2820                 :      */
        2821           19675 :     if (!shrinking && shrinkFlag) {
        2822               0 :         shrinkFlag = false;
        2823               0 :         ExpireChunksAndArenas(rt, true);
        2824                 :     }
        2825           19675 : }
        2826                 :
        2827                 : #endif /* JS_THREADSAFE */
        2828                 :
        2829                 : } /* namespace js */
        2830                 :
        2831                 : static bool
        2832           38427 : ReleaseObservedTypes(JSRuntime *rt)
        2833                 : {
        2834           38427 :     bool releaseTypes = false;
        2835           38427 :     int64_t now = PRMJ_Now();
        2836           38427 :     if (now >= rt->gcJitReleaseTime) {
        2837               4 :         releaseTypes = true;
        2838               4 :         rt->gcJitReleaseTime = now + JIT_SCRIPT_RELEASE_TYPES_INTERVAL;
        2839                 :     }
        2840                 :
        2841           38427 :     return releaseTypes;
        2842                 : }
        2843                 :
        2844                 : static void
        2845           38427 : SweepCompartments(FreeOp *fop, JSGCInvocationKind gckind)
        2846                 : {
        2847           38427 :     JSRuntime *rt = fop->runtime();
        2848           38427 :     JSDestroyCompartmentCallback callback = rt->destroyCompartmentCallback;
        2849                 :
        2850                 :     /* Skip the atomsCompartment. */
        2851           38427 :     JSCompartment **read = rt->compartments.begin() + 1;
        2852           38427 :     JSCompartment **end = rt->compartments.end();
        2853           38427 :     JSCompartment **write = read;
        2854           38427 :     JS_ASSERT(rt->compartments.length() >= 1);
        2855           38427 :     JS_ASSERT(*rt->compartments.begin() == rt->atomsCompartment);
        2856                 :
        2857          122385 :     while (read < end) {
        2858           45531 :         JSCompartment *compartment = *read++;
        2859                 :
        2860          113835 :         if (!compartment->hold && compartment->isCollecting() &&
        2861           68304 :             (compartment->arenas.arenaListsAreEmpty() || !rt->hasContexts()))
        2862                 :         {
        2863           22524 :             compartment->arenas.checkEmptyFreeLists();
        2864           22524 :             if (callback)
        2865               0 :                 callback(fop, compartment);
        2866           22524 :             if (compartment->principals)
        2867               3 :                 JS_DropPrincipals(rt, compartment->principals);
        2868           22524 :             fop->delete_(compartment);
        2869           22524 :             continue;
        2870                 :         }
        2871           23007 :         *write++ = compartment;
        2872                 :     }
        2873           38427 :     rt->compartments.resize(write - rt->compartments.begin());
        2874           38427 : }
        2875                 :
        2876                 : static void
        2877           39964 : PurgeRuntime(JSTracer *trc)
        2878                 : {
        2879           39964 :     JSRuntime *rt = trc->runtime;
        2880                 :
        2881          129330 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        2882                 :         /* We can be called from StartVerifyBarriers with a non-GC marker. */
        2883           89366 :         if (c->isCollecting() || !IS_GC_MARKING_TRACER(trc))
        2884           89042 :             c->purge();
        2885                 :     }
        2886                 :
        2887           39964 :     rt->tempLifoAlloc.freeUnused();
        2888           39964 :     rt->gsnCache.purge();
        2889                 :
        2890                 :     /* FIXME: bug 506341 */
        2891           39964 :     rt->propertyCache.purge(rt);
        2892                 :
        2893           61067 :     for (ContextIter acx(rt); !acx.done(); acx.next())
        2894           21103 :         acx->purge();
        2895           39964 : }
        2896                 :
        2897                 : static void
        2898           38454 : BeginMarkPhase(JSRuntime *rt)
        2899                 : {
        2900           38454 :     GCMarker *gcmarker = &rt->gcMarker;
        2901                 :
        2902           38454 :     rt->gcStartNumber = rt->gcNumber;
        2903                 :
        2904                 :     /* Reset weak map list. */
        2905           38454 :     WeakMapBase::resetWeakMapList(rt);
        2906                 :
        2907                 :     /*
        2908                 :      * We must purge the runtime at the beginning of an incremental GC. The
        2909                 :      * danger if we purge later is that the snapshot invariant of incremental
        2910                 :      * GC will be broken, as follows. If some object is reachable only through
        2911                 :      * some cache (say the dtoaCache) then it will not be part of the snapshot.
        2912                 :      * If we purge after root marking, then the mutator could obtain a pointer
        2913                 :      * to the object and start using it. This object might never be marked, so
        2914                 :      * a GC hazard would exist.
        2915                 :      */
        2916                 :     {
        2917           76908 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_PURGE);
        2918           38454 :         PurgeRuntime(gcmarker);
        2919                 :     }
        2920                 :
        2921                 :     /*
        2922                 :      * Mark phase.
        2923                 :      */
        2924           76908 :     gcstats::AutoPhase ap1(rt->gcStats, gcstats::PHASE_MARK);
        2925           76908 :     gcstats::AutoPhase ap2(rt->gcStats, gcstats::PHASE_MARK_ROOTS);
        2926                 :
        2927          118908 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
        2928           80454 :         r.front()->bitmap.clear();
        2929                 :
        2930           38454 :     MarkRuntime(gcmarker);
        2931           38454 : }
        2932                 :
        2933                 : void
        2934           76872 : MarkWeakReferences(GCMarker *gcmarker)
        2935                 : {
        2936           76872 :     JS_ASSERT(gcmarker->isDrained());
        2937          308200 :     while (WatchpointMap::markAllIteratively(gcmarker) ||
        2938           77192 :            WeakMapBase::markAllIteratively(gcmarker) ||
        2939           76944 :            Debugger::markAllIteratively(gcmarker))
        2940                 :     {
        2941             320 :         SliceBudget budget;
        2942             320 :         gcmarker->drainMarkStack(budget);
        2943                 :     }
        2944           76872 :     JS_ASSERT(gcmarker->isDrained());
        2945           76872 : }
        2946                 :
        2947                 : static void
        2948           38436 : MarkGrayAndWeak(JSRuntime *rt)
        2949                 : {
        2950           38436 :     GCMarker *gcmarker = &rt->gcMarker;
        2951                 :
        2952           38436 :     JS_ASSERT(gcmarker->isDrained());
        2953           38436 :     MarkWeakReferences(gcmarker);
        2954                 :
        2955           38436 :     gcmarker->setMarkColorGray();
        2956           38436 :     if (gcmarker->hasBufferedGrayRoots()) {
        2957           38436 :         gcmarker->markBufferedGrayRoots();
        2958                 :     } else {
        2959               0 :         if (JSTraceDataOp op = rt->gcGrayRootsTraceOp)
        2960               0 :             (*op)(gcmarker, rt->gcGrayRootsData);
        2961                 :     }
        2962           38436 :     SliceBudget budget;
        2963           38436 :     gcmarker->drainMarkStack(budget);
        2964           38436 :     MarkWeakReferences(gcmarker);
        2965           38436 :     JS_ASSERT(gcmarker->isDrained());
        2966           38436 : }
        2967                 :
        2968                 : #ifdef DEBUG
        2969                 : static void
        2970                 : ValidateIncrementalMarking(JSContext *cx);
        2971                 : #endif
        2972                 :
        2973                 : static void
        2974           38427 : EndMarkPhase(JSContext *cx)
        2975                 : {
        2976           38427 :     JSRuntime *rt = cx->runtime;
        2977                 :
        2978                 :     {
        2979           76854 :         gcstats::AutoPhase ap1(rt->gcStats, gcstats::PHASE_MARK);
        2980           76854 :         gcstats::AutoPhase ap2(rt->gcStats, gcstats::PHASE_MARK_OTHER);
        2981           38427 :         MarkGrayAndWeak(rt);
        2982                 :     }
        2983                 :
        2984           38427 :     JS_ASSERT(rt->gcMarker.isDrained());
        2985                 :
        2986                 : #ifdef DEBUG
        2987           38427 :     if (rt->gcIncrementalState != NO_INCREMENTAL)
        2988               9 :         ValidateIncrementalMarking(cx);
        2989                 : #endif
        2990                 :
        2991                 : #ifdef DEBUG
        2992                 :     /* Make sure that we didn't mark an object in another compartment */
        2993          122385 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        2994           84336 :         JS_ASSERT_IF(!c->isCollecting() && c != rt->atomsCompartment,
        2995           84336 :                      c->arenas.checkArenaListAllUnmarked());
        2996                 :     }
        2997                 : #endif
        2998           38427 : }
        2999                 :
        3000                 : #ifdef DEBUG
        3001                 : static void
        3002               9 : ValidateIncrementalMarking(JSContext *cx)
        3003                 : {
        3004                 :     typedef HashMap<Chunk *, uintptr_t *, GCChunkHasher, SystemAllocPolicy> BitmapMap;
        3005              18 :     BitmapMap map;
        3006               9 :     if (!map.init())
        3007                 :         return;
        3008                 :
        3009               9 :     JSRuntime *rt = cx->runtime;
        3010               9 :     GCMarker *gcmarker = &rt->gcMarker;
        3011                 :
        3012                 :     /* Save existing mark bits. */
        3013              27 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront()) {
        3014              18 :         ChunkBitmap *bitmap = &r.front()->bitmap;
        3015              18 :         uintptr_t *entry = (uintptr_t *)js_malloc(sizeof(bitmap->bitmap));
        3016              18 :         if (!entry)
        3017                 :             return;
        3018                 :
        3019              18 :         memcpy(entry, bitmap->bitmap, sizeof(bitmap->bitmap));
        3020              18 :         if (!map.putNew(r.front(), entry))
        3021                 :             return;
        3022                 :     }
        3023                 :
        3024                 :     /* Save the existing weakmaps. */
        3025              18 :     WeakMapVector weakmaps;
        3026               9 :     if (!WeakMapBase::saveWeakMapList(rt, weakmaps))
        3027                 :         return;
        3028                 :
        3029                 :     /*
        3030                 :      * After this point, the function should run to completion, so we shouldn't
        3031                 :      * do anything fallible.
        3032                 :      */
        3033                 :
        3034                 :     /* Re-do all the marking, but non-incrementally. */
        3035               9 :     js::gc::State state = rt->gcIncrementalState;
        3036               9 :     rt->gcIncrementalState = NO_INCREMENTAL;
        3037                 :
        3038                 :     /* As we're re-doing marking, we need to reset the weak map list. */
        3039               9 :     WeakMapBase::resetWeakMapList(rt);
        3040                 :
        3041               9 :     JS_ASSERT(gcmarker->isDrained());
        3042               9 :     gcmarker->reset();
        3043                 :
        3044              27 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
        3045              18 :         r.front()->bitmap.clear();
        3046                 :
        3047               9 :     MarkRuntime(gcmarker, true);
        3048               9 :     SliceBudget budget;
        3049               9 :     rt->gcMarker.drainMarkStack(budget);
        3050               9 :     MarkGrayAndWeak(rt);
        3051                 :
        3052                 :     /* Now verify that we have the same mark bits as before. */
        3053              27 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront()) {
        3054              18 :         Chunk *chunk = r.front();
        3055              18 :         ChunkBitmap *bitmap = &chunk->bitmap;
        3056              18 :         uintptr_t *entry = map.lookup(r.front())->value;
        3057                 :         ChunkBitmap incBitmap;
        3058                 :
        3059              18 :         memcpy(incBitmap.bitmap, entry, sizeof(incBitmap.bitmap));
        3060              18 :         js_free(entry);
        3061                 :
        3062            4554 :         for (size_t i = 0; i < ArenasPerChunk; i++) {
        3063            4536 :             if (chunk->decommittedArenas.get(i))
        3064               0 :                 continue;
        3065            4536 :             Arena *arena = &chunk->arenas[i];
        3066            4536 :             if (!arena->aheader.allocated())
        3067            4158 :                 continue;
        3068             378 :             if (!arena->aheader.compartment->isCollecting())
        3069               0 :                 continue;
        3070             378 :             if (arena->aheader.allocatedDuringIncremental)
        3071               0 :                 continue;
        3072                 :
        3073             378 :             AllocKind kind = arena->aheader.getAllocKind();
        3074             378 :             uintptr_t thing = arena->thingsStart(kind);
        3075             378 :             uintptr_t end = arena->thingsEnd();
        3076           72432 :             while (thing < end) {
        3077           71676 :                 Cell *cell = (Cell *)thing;
        3078           71676 :                 JS_ASSERT_IF(bitmap->isMarked(cell, BLACK), incBitmap.isMarked(cell, BLACK));
        3079           71676 :                 thing += Arena::thingSize(kind);
        3080                 :             }
        3081                 :         }
        3082                 :
        3083              18 :         memcpy(bitmap->bitmap, incBitmap.bitmap, sizeof(incBitmap.bitmap));
        3084                 :     }
        3085                 :
        3086                 :     /* Restore the weak map list. */
        3087               9 :     WeakMapBase::resetWeakMapList(rt);
        3088               9 :     WeakMapBase::restoreWeakMapList(rt, weakmaps);
        3089                 :
        3090              18 :     rt->gcIncrementalState = state;
        3091                 : }
        3092                 : #endif
        3093                 :
        3094                 : static void
        3095           38427 : SweepPhase(JSContext *cx, JSGCInvocationKind gckind)
        3096                 : {
        3097           38427 :     JSRuntime *rt = cx->runtime;
        3098                 :
        3099                 :     /*
        3100                 :      * Sweep phase.
        3101                 :      *
        3102                 :      * Finalize as we sweep, outside of rt->gcLock but with rt->gcRunning set
        3103                 :      * so that any attempt to allocate a GC-thing from a finalizer will fail,
        3104                 :      * rather than nest badly and leave the unmarked newborn to be swept.
        3105                 :      *
        3106                 :      * We first sweep atom state so we can use IsAboutToBeFinalized on
        3107                 :      * JSString held in a hashtable to check if the hashtable entry can be
        3108                 :      * freed. Note that even after the entry is freed, JSObject finalizers can
        3109                 :      * continue to access the corresponding JSString* assuming that they are
        3110                 :      * unique. This works since the atomization API must not be called during
        3111                 :      * the GC.
        3112                 :      */
        3113           76854 :     gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP);
        3114                 :
        3115                 : #ifdef JS_THREADSAFE
        3116           38427 :     if (rt->hasContexts() && rt->gcHelperThread.prepareForBackgroundSweep())
        3117           19666 :         cx->gcBackgroundFree = &rt->gcHelperThread;
        3118                 : #endif
        3119                 :
        3120                 :     /* Purge the ArenaLists before sweeping. */
        3121          122124 :     for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3122           83697 :         c->arenas.purge();
        3123                 :
        3124                 : #ifdef JS_THREADSAFE
        3125           38427 :     FreeOp fop(rt, !!cx->gcBackgroundFree, false);
        3126                 : #else
        3127                 :     FreeOp fop(rt, false, false);
        3128                 : #endif
        3129                 :     {
        3130           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_FINALIZE_START);
        3131           38427 :         if (rt->gcFinalizeCallback)
        3132               2 :             rt->gcFinalizeCallback(&fop, JSFINALIZE_START);
        3133                 :     }
        3134                 :
        3135                 :     /* Finalize unreachable (key,value) pairs in all weak maps. */
        3136           38427 :     WeakMapBase::sweepAll(&rt->gcMarker);
        3137                 :
        3138           38427 :     SweepAtomState(rt);
        3139                 :
        3140                 :     /* Collect watch points associated with unreachable objects. */
        3141           38427 :     WatchpointMap::sweepAll(rt);
        3142                 :
        3143                 :     /* Detach unreachable debuggers and global objects from each other. */
        3144           38427 :     Debugger::sweepAll(&fop);
        3145                 :
        3146                 :     {
        3147           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP_COMPARTMENTS);
        3148                 :
        3149           38427 :         bool releaseTypes = ReleaseObservedTypes(rt);
        3150          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3151           83697 :             c->sweep(&fop, releaseTypes);
        3152                 :     }
        3153                 :
        3154                 :     {
        3155           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP_OBJECT);
        3156                 :
        3157                 :         /*
        3158                 :          * We finalize objects before other GC things to ensure that the object's
        3159                 :          * finalizer can access the other things even if they will be freed.
        3160                 :          */
        3161          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3162           83697 :             c->arenas.finalizeObjects(&fop);
        3163                 :     }
        3164                 :
        3165                 :     {
        3166           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP_STRING);
        3167          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3168           83697 :             c->arenas.finalizeStrings(&fop);
        3169                 :     }
        3170                 :
        3171                 :     {
        3172           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP_SCRIPT);
        3173          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3174           83697 :             c->arenas.finalizeScripts(&fop);
        3175                 :     }
        3176                 :
        3177                 :     {
        3178           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_SWEEP_SHAPE);
        3179          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3180           83697 :             c->arenas.finalizeShapes(&fop);
        3181                 :     }
        3182                 :
        3183                 : #ifdef DEBUG
        3184           38427 :      PropertyTree::dumpShapes(rt);
        3185                 : #endif
        3186                 :
        3187                 :     {
        3188           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_DESTROY);
        3189                 :
        3190                 :         /*
        3191                 :          * Sweep script filenames after sweeping functions in the generic loop
        3192                 :          * above. In this way when a scripted function's finalizer destroys the
        3193                 :          * script and calls rt->destroyScriptHook, the hook can still access the
        3194                 :          * script's filename. See bug 323267.
        3195                 :          */
        3196          122124 :         for (GCCompartmentsIter c(rt); !c.done(); c.next())
        3197           83697 :             SweepScriptFilenames(c);
        3198                 :
        3199                 :         /*
        3200                 :          * This removes compartments from rt->compartment, so we do it last to make
        3201                 :          * sure we don't miss sweeping any compartments.
        3202                 :          */
        3203           38427 :         SweepCompartments(&fop, gckind);
        3204                 :
        3205                 : #ifndef JS_THREADSAFE
        3206                 :         /*
        3207                 :          * Destroy arenas after we finished the sweeping so finalizers can safely
        3208                 :          * use IsAboutToBeFinalized().
        3209                 :          * This is done on the GCHelperThread if JS_THREADSAFE is defined.
        3210                 :          */
        3211                 :         ExpireChunksAndArenas(rt, gckind == GC_SHRINK);
        3212                 : #endif
        3213                 :     }
        3214                 :
        3215                 :     {
        3216           76854 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_FINALIZE_END);
        3217           38427 :         if (rt->gcFinalizeCallback)
        3218               2 :             rt->gcFinalizeCallback(&fop, JSFINALIZE_END);
        3219                 :     }
        3220                 :
        3221           99861 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        3222           61434 :         c->setGCLastBytes(c->gcBytes, c->gcMallocAndFreeBytes, gckind);
        3223           38427 : }
        3224                 :
        3225                 : /* Perform mark-and-sweep GC. If comp is set, we perform a single-compartment GC. */
        3226                 : static void
        3227           38418 : MarkAndSweep(JSContext *cx, JSGCInvocationKind gckind)
        3228                 : {
        3229           38418 :     JSRuntime *rt = cx->runtime;
        3230                 :
        3231           76836 :     AutoUnlockGC unlock(rt);
        3232                 :
        3233           38418 :     rt->gcMarker.start(rt);
        3234           38418 :     JS_ASSERT(!rt->gcMarker.callback);
        3235                 :
        3236           38418 :     BeginMarkPhase(rt);
        3237                 :     {
        3238           76836 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_MARK);
        3239           38418 :         SliceBudget budget;
        3240           38418 :         rt->gcMarker.drainMarkStack(budget);
        3241                 :     }
        3242           38418 :     EndMarkPhase(cx);
        3243           38418 :     SweepPhase(cx, gckind);
        3244                 :
        3245           38418 :     rt->gcMarker.stop();
        3246           38418 : }
        3247                 :
        3248                 : /*
        3249                 :  * This class should be used by any code that needs to exclusive access to the
        3250                 :  * heap in order to trace through it...
        3251                 :  */
        3252                 : class AutoHeapSession {
        3253                 :   public:
        3254                 :     explicit AutoHeapSession(JSRuntime *rt);
        3255                 :     ~AutoHeapSession();
        3256                 :
        3257                 :   protected:
        3258                 :     JSRuntime *runtime;
        3259                 :
        3260                 :   private:
        3261                 :     AutoHeapSession(const AutoHeapSession&) MOZ_DELETE;
        3262                 :     void operator=(const AutoHeapSession&) MOZ_DELETE;
        3263                 : };
        3264                 :
        3265                 : /* ...while this class is to be used only for garbage collection. */
        3266                 : class AutoGCSession : AutoHeapSession {
        3267                 :   public:
        3268                 :     explicit AutoGCSession(JSRuntime *rt);
        3269                 :     ~AutoGCSession();
        3270                 : };
        3271                 :
        3272                 : /* Start a new heap session. */
        3273           42080 : AutoHeapSession::AutoHeapSession(JSRuntime *rt)
        3274           42080 :   : runtime(rt)
        3275                 : {
        3276           42080 :     JS_ASSERT(!rt->noGCOrAllocationCheck);
        3277           42080 :     JS_ASSERT(!rt->gcRunning);
        3278           42080 :     rt->gcRunning = true;
        3279           42080 : }
        3280                 :
        3281           42080 : AutoHeapSession::~AutoHeapSession()
        3282                 : {
        3283           42080 :     JS_ASSERT(runtime->gcRunning);
        3284           42080 :     runtime->gcRunning = false;
        3285           42080 : }
        3286                 :
        3287           38481 : AutoGCSession::AutoGCSession(JSRuntime *rt)
        3288           38481 :   : AutoHeapSession(rt)
        3289                 : {
        3290           76962 :     DebugOnly<bool> any = false;
        3291          122619 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3292           84138 :         if (c->isGCScheduled()) {
        3293           83796 :             c->setCollecting(true);
        3294           83796 :             any = true;
        3295                 :         }
        3296                 :     }
        3297           38481 :     JS_ASSERT(any);
        3298                 :
        3299           38481 :     runtime->gcIsNeeded = false;
        3300           38481 :     runtime->gcInterFrameGC = true;
        3301                 :
        3302           38481 :     runtime->gcNumber++;
        3303                 :
        3304           38481 :     runtime->resetGCMallocBytes();
        3305                 :
        3306                 :     /* Clear gcMallocBytes for all compartments */
        3307          122619 :     for (CompartmentsIter c(runtime); !c.done(); c.next())
        3308           84138 :         c->resetGCMallocBytes();
        3309           38481 : }
        3310                 :
        3311           76962 : AutoGCSession::~AutoGCSession()
        3312                 : {
        3313           99753 :     for (GCCompartmentsIter c(runtime); !c.done(); c.next())
        3314           61272 :         c->setCollecting(false);
        3315                 :
        3316           38481 :     runtime->gcNextFullGCTime = PRMJ_Now() + GC_IDLE_FULL_SPAN;
        3317           38481 :     runtime->gcChunkAllocationSinceLastGC = false;
        3318                 :
        3319                 : #ifdef JS_GC_ZEAL
        3320                 :     /* Keeping these around after a GC is dangerous. */
        3321           38481 :     runtime->gcSelectedForMarking.clearAndFree();
        3322                 : #endif
        3323           38481 : }
        3324                 :
        3325                 : static void
        3326           38454 : ResetIncrementalGC(JSRuntime *rt, const char *reason)
        3327                 : {
        3328           38454 :     if (rt->gcIncrementalState == NO_INCREMENTAL)
        3329           38427 :         return;
        3330                 :
        3331             135 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        3332             108 :         c->needsBarrier_ = false;
        3333                 :
        3334              27 :     rt->gcMarker.reset();
        3335              27 :     rt->gcMarker.stop();
        3336              27 :     rt->gcIncrementalState = NO_INCREMENTAL;
        3337                 :
        3338              27 :     JS_ASSERT(!rt->gcStrictCompartmentChecking);
        3339                 :
        3340              27 :     rt->gcStats.reset(reason);
        3341                 : }
        3342                 :
        3343                 : class AutoGCSlice {
        3344                 :   public:
        3345                 :     AutoGCSlice(JSContext *cx);
        3346                 :     ~AutoGCSlice();
        3347                 :
        3348                 :   private:
        3349                 :     JSContext *context;
        3350                 : };
        3351                 :
        3352              63 : AutoGCSlice::AutoGCSlice(JSContext *cx)
        3353              63 :   : context(cx)
        3354                 : {
        3355              63 :     JSRuntime *rt = context->runtime;
        3356                 :
        3357                 :     /*
        3358                 :      * During incremental GC, the compartment's active flag determines whether
        3359                 :      * there are stack frames active for any of its scripts. Normally this flag
        3360                 :      * is set at the beginning of the mark phase. During incremental GC, we also
        3361                 :      * set it at the start of every phase.
        3362                 :      */
        3363              63 :     rt->stackSpace.markActiveCompartments();
        3364                 :
        3365             180 :     for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
        3366                 :         /* Clear this early so we don't do any write barriers during GC. */
        3367             117 :         if (rt->gcIncrementalState == MARK)
        3368              54 :             c->needsBarrier_ = false;
        3369                 :         else
        3370              63 :             JS_ASSERT(!c->needsBarrier_);
        3371                 :     }
        3372              63 : }
        3373                 :
        3374              63 : AutoGCSlice::~AutoGCSlice()
        3375                 : {
        3376              63 :     JSRuntime *rt = context->runtime;
        3377                 :
        3378             180 :     for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
        3379             117 :         if (rt->gcIncrementalState == MARK) {
        3380              99 :             c->needsBarrier_ = true;
        3381              99 :             c->arenas.prepareForIncrementalGC(rt);
        3382                 :         } else {
        3383              18 :             JS_ASSERT(rt->gcIncrementalState == NO_INCREMENTAL);
        3384              18 :             c->needsBarrier_ = false;
        3385                 :         }
        3386                 :     }
        3387              63 : }
        3388                 :
        3389                 : class AutoCopyFreeListToArenas {
        3390                 :     JSRuntime *rt;
        3391                 :
        3392                 :   public:
        3393           41950 :     AutoCopyFreeListToArenas(JSRuntime *rt)
        3394           41950 :       : rt(rt) {
        3395          137767 :         for (CompartmentsIter c(rt); !c.done(); c.next())
        3396           95817 :             c->arenas.copyFreeListsToArenas();
        3397           41950 :     }
        3398                 :
        3399           41950 :     ~AutoCopyFreeListToArenas() {
        3400          115243 :         for (CompartmentsIter c(rt); !c.done(); c.next())
        3401           73293 :             c->arenas.clearFreeListsInArenas();
        3402           41950 :     }
        3403                 : };
        3404                 :
        3405                 : static void
        3406              63 : IncrementalGCSlice(JSContext *cx, int64_t budget, JSGCInvocationKind gckind)
        3407                 : {
        3408              63 :     JSRuntime *rt = cx->runtime;
        3409                 :
        3410             126 :     AutoUnlockGC unlock(rt);
        3411             126 :     AutoGCSlice slice(cx);
        3412                 :
        3413              63 :     gc::State initialState = rt->gcIncrementalState;
        3414                 :
        3415              63 :     if (rt->gcIncrementalState == NO_INCREMENTAL) {
        3416              36 :         rt->gcIncrementalState = MARK_ROOTS;
        3417              36 :         rt->gcLastMarkSlice = false;
        3418                 :     }
        3419                 :
        3420              63 :     if (rt->gcIncrementalState == MARK_ROOTS) {
        3421              36 :         rt->gcMarker.start(rt);
        3422              36 :         JS_ASSERT(IS_GC_MARKING_TRACER(&rt->gcMarker));
        3423                 :
        3424              99 :         for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
        3425             126 :             gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_DISCARD_CODE);
        3426              63 :             c->discardJitCode(rt->defaultFreeOp());
        3427                 :         }
        3428                 :
        3429              36 :         BeginMarkPhase(rt);
        3430                 :
        3431              36 :         rt->gcIncrementalState = MARK;
        3432                 :     }
        3433                 :
        3434              63 :     if (rt->gcIncrementalState == MARK) {
        3435             126 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_MARK);
        3436              63 :         SliceBudget sliceBudget(budget);
        3437                 :
        3438                 :         /* If we needed delayed marking for gray roots, then collect until done. */
        3439              63 :         if (!rt->gcMarker.hasBufferedGrayRoots())
        3440               0 :             sliceBudget.reset();
        3441                 :
        3442                 : #ifdef JS_GC_ZEAL
        3443              63 :         if (!rt->gcSelectedForMarking.empty()) {
        3444              36 :             for (JSObject **obj = rt->gcSelectedForMarking.begin();
        3445              18 :                  obj != rt->gcSelectedForMarking.end(); obj++)
        3446                 :             {
        3447               9 :                 MarkObjectUnbarriered(&rt->gcMarker, obj, "selected obj");
        3448                 :             }
        3449                 :         }
        3450                 : #endif
        3451                 :
        3452              63 :         bool finished = rt->gcMarker.drainMarkStack(sliceBudget);
        3453                 :
        3454              63 :         if (finished) {
        3455               9 :             JS_ASSERT(rt->gcMarker.isDrained());
        3456               9 :             if (initialState == MARK && !rt->gcLastMarkSlice && budget != SliceBudget::Unlimited)
        3457               0 :                 rt->gcLastMarkSlice = true;
        3458                 :             else
        3459               9 :                 rt->gcIncrementalState = SWEEP;
        3460                 :         }
        3461                 :     }
        3462                 :
        3463              63 :     if (rt->gcIncrementalState == SWEEP) {
        3464               9 :         EndMarkPhase(cx);
        3465               9 :         SweepPhase(cx, gckind);
        3466                 :
        3467               9 :         rt->gcMarker.stop();
        3468                 :
        3469                 :         /* JIT code was already discarded during sweeping. */
        3470                 :
        3471               9 :         rt->gcIncrementalState = NO_INCREMENTAL;
        3472                 :     }
        3473              63 : }
        3474                 :
        3475                 : class IncrementalSafety
        3476                 : {
        3477                 :     const char *reason_;
        3478                 :
        3479            3243 :     IncrementalSafety(const char *reason) : reason_(reason) {}
        3480                 :
        3481                 :   public:
        3482            3065 :     static IncrementalSafety Safe() { return IncrementalSafety(NULL); }
        3483             178 :     static IncrementalSafety Unsafe(const char *reason) { return IncrementalSafety(reason); }
        3484                 :
        3485                 :     typedef void (IncrementalSafety::* ConvertibleToBool)();
        3486               0 :     void nonNull() {}
        3487                 :
        3488            3243 :     operator ConvertibleToBool() const {
        3489            3243 :         return reason_ == NULL ? &IncrementalSafety::nonNull : 0;
        3490                 :     }
        3491                 :
        3492               0 :     const char *reason() {
        3493               0 :         JS_ASSERT(reason_);
        3494               0 :         return reason_;
        3495                 :     }
        3496                 : };
        3497                 :
        3498                 : static IncrementalSafety
        3499            3243 : IsIncrementalGCSafe(JSRuntime *rt)
        3500                 : {
        3501            3243 :     if (rt->gcKeepAtoms)
        3502             178 :         return IncrementalSafety::Unsafe("gcKeepAtoms set");
        3503                 :
        3504           13756 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3505           10691 :         if (c->activeAnalysis)
        3506               0 :             return IncrementalSafety::Unsafe("activeAnalysis set");
        3507                 :     }
        3508                 :
        3509            3065 :     if (!rt->gcIncrementalEnabled)
        3510               0 :         return IncrementalSafety::Unsafe("incremental permanently disabled");
        3511                 :
        3512            3065 :     return IncrementalSafety::Safe();
        3513                 : }
        3514                 :
        3515                 : static void
        3516             241 : BudgetIncrementalGC(JSRuntime *rt, int64_t *budget)
        3517                 : {
        3518             241 :     IncrementalSafety safe = IsIncrementalGCSafe(rt);
        3519             241 :     if (!safe) {
        3520               0 :         ResetIncrementalGC(rt, safe.reason());
        3521               0 :         *budget = SliceBudget::Unlimited;
        3522               0 :         rt->gcStats.nonincremental(safe.reason());
        3523               0 :         return;
        3524                 :     }
        3525                 :
        3526             241 :     if (rt->gcMode != JSGC_MODE_INCREMENTAL) {
        3527               0 :         ResetIncrementalGC(rt, "GC mode change");
        3528               0 :         *budget = SliceBudget::Unlimited;
        3529               0 :         rt->gcStats.nonincremental("GC mode");
        3530               0 :         return;
        3531                 :     }
        3532                 :
        3533                 : #ifdef ANDROID
        3534                 :     JS_ASSERT(rt->gcIncrementalState == NO_INCREMENTAL);
        3535                 :     *budget = SliceBudget::Unlimited;
        3536                 :     rt->gcStats.nonincremental("Android");
        3537                 :     return;
        3538                 : #endif
        3539                 :
        3540             403 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3541             376 :         if (c->gcBytes > c->gcTriggerBytes) {
        3542               0 :             *budget = SliceBudget::Unlimited;
        3543               0 :             rt->gcStats.nonincremental("allocation trigger");
        3544               0 :             return;
        3545                 :         }
        3546                 :
        3547             376 :         if (c->isCollecting() != c->needsBarrier()) {
        3548             214 :             ResetIncrementalGC(rt, "compartment change");
        3549             214 :             return;
        3550                 :         }
        3551                 :     }
        3552                 : }
        3553                 :
        3554                 : /*
        3555                 :  * GC, repeatedly if necessary, until we think we have not created any new
        3556                 :  * garbage. We disable inlining to ensure that the bottom of the stack with
        3557                 :  * possible GC roots recorded in MarkRuntime excludes any pointers we use during
        3558                 :  * the marking implementation.
        3559                 :  */
        3560                 : static JS_NEVER_INLINE void
        3561           38481 : GCCycle(JSContext *cx, bool incremental, int64_t budget, JSGCInvocationKind gckind)
        3562                 : {
        3563           38481 :     JSRuntime *rt = cx->runtime;
        3564                 :
        3565                 : #ifdef DEBUG
        3566          122619 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        3567           84138 :         JS_ASSERT_IF(rt->gcMode == JSGC_MODE_GLOBAL, c->isGCScheduled());
        3568                 : #endif
        3569                 :
        3570                 :     /* Recursive GC is no-op. */
        3571           38481 :     if (rt->gcRunning)
        3572               0 :         return;
        3573                 :
        3574           76962 :     AutoGCSession gcsession(rt);
        3575                 :
        3576                 :     /* Don't GC if we are reporting an OOM. */
        3577           38481 :     if (rt->inOOMReport)
        3578                 :         return;
        3579                 :
        3580                 : #ifdef JS_THREADSAFE
        3581                 :     /*
        3582                 :      * As we about to purge caches and clear the mark bits we must wait for
        3583                 :      * any background finalization to finish. We must also wait for the
        3584                 :      * background allocation to finish so we can avoid taking the GC lock
        3585                 :      * when manipulating the chunks during the GC.
        3586                 :      */
        3587                 :     {
        3588           76962 :         gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_WAIT_BACKGROUND_THREAD);
        3589                 :
        3590           38481 :         JS_ASSERT(!cx->gcBackgroundFree);
        3591           38481 :         rt->gcHelperThread.waitBackgroundSweepOrAllocEnd();
        3592                 :     }
        3593                 : #endif
        3594                 :
        3595           38481 :     if (!incremental) {
        3596                 :         /* If non-incremental GC was requested, reset incremental GC. */
        3597           38240 :         ResetIncrementalGC(rt, "requested");
        3598           38240 :         rt->gcStats.nonincremental("requested");
        3599                 :     } else {
        3600             241 :         BudgetIncrementalGC(rt, &budget);
        3601                 :     }
        3602                 :
        3603           76962 :     AutoCopyFreeListToArenas copy(rt);
        3604                 :
        3605           38481 :     if (budget == SliceBudget::Unlimited && rt->gcIncrementalState == NO_INCREMENTAL)
        3606           38418 :         MarkAndSweep(cx, gckind);
        3607                 :     else
        3608              63 :         IncrementalGCSlice(cx, budget, gckind);
        3609                 :
        3610                 : #ifdef DEBUG
        3611           38481 :     if (rt->gcIncrementalState == NO_INCREMENTAL) {
        3612           99861 :         for (CompartmentsIter c(rt); !c.done(); c.next())
        3613           61434 :             JS_ASSERT(!c->needsBarrier_);
        3614                 :     }
        3615                 : #endif
        3616                 : #ifdef JS_THREADSAFE
        3617           38481 :     if (rt->gcIncrementalState == NO_INCREMENTAL) {
        3618           38427 :         if (cx->gcBackgroundFree) {
        3619           19666 :             JS_ASSERT(cx->gcBackgroundFree == &rt->gcHelperThread);
        3620           19666 :             cx->gcBackgroundFree = NULL;
        3621           19666 :             rt->gcHelperThread.startBackgroundSweep(cx, gckind == GC_SHRINK);
        3622                 :         }
        3623                 :     }
        3624                 : #endif
        3625                 : }
        3626                 :
        3627                 : #ifdef JS_GC_ZEAL
        3628                 : static bool
        3629               0 : IsDeterministicGCReason(gcreason::Reason reason)
        3630                 : {
        3631               0 :     if (reason > gcreason::DEBUG_GC && reason != gcreason::CC_FORCED)
        3632               0 :         return false;
        3633                 :
        3634               0 :     if (reason == gcreason::MAYBEGC)
        3635               0 :         return false;
        3636                 :
        3637               0 :     return true;
        3638                 : }
        3639                 : #endif
        3640                 :
        3641                 : static void
        3642           38481 : Collect(JSContext *cx, bool incremental, int64_t budget,
        3643                 :         JSGCInvocationKind gckind, gcreason::Reason reason)
        3644                 : {
        3645           38481 :     JSRuntime *rt = cx->runtime;
        3646           38481 :     JS_AbortIfWrongThread(rt);
        3647                 :
        3648                 : #ifdef JS_GC_ZEAL
        3649           38481 :     if (rt->gcDeterministicOnly && !IsDeterministicGCReason(reason))
        3650               0 :         return;
        3651                 : #endif
        3652                 :
        3653                 :     JS_ASSERT_IF(!incremental || budget != SliceBudget::Unlimited, JSGC_INCREMENTAL);
        3654                 :
        3655                 : #ifdef JS_GC_ZEAL
        3656                 :     bool restartVerify = cx->runtime->gcVerifyData &&
        3657             130 :                          cx->runtime->gcZeal() == ZealVerifierValue &&
        3658           38611 :                          reason != gcreason::CC_FORCED;
        3659                 :
        3660                 :     struct AutoVerifyBarriers {
        3661                 :         JSContext *cx;
        3662                 :         bool restart;
        3663           38481 :         AutoVerifyBarriers(JSContext *cx, bool restart)
        3664           38481 :           : cx(cx), restart(restart)
        3665                 :         {
        3666           38481 :             if (cx->runtime->gcVerifyData)
        3667             130 :                 EndVerifyBarriers(cx);
        3668           38481 :         }
        3669           38481 :         ~AutoVerifyBarriers() {
        3670           38481 :             if (restart)
        3671             130 :                 StartVerifyBarriers(cx);
        3672           38481 :         }
        3673           76962 :     } av(cx, restartVerify);
        3674                 : #endif
        3675                 :
        3676           38481 :     RecordNativeStackTopForGC(rt);
        3677                 :
        3678           38481 :     int compartmentCount = 0;
        3679           38481 :     int collectedCount = 0;
        3680          122619 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3681           84138 :         if (rt->gcMode == JSGC_MODE_GLOBAL)
        3682             303 :             c->scheduleGC();
        3683                 :
        3684                 :         /* This is a heuristic to avoid resets. */
        3685           84138 :         if (rt->gcIncrementalState != NO_INCREMENTAL && c->needsBarrier())
        3686              99 :             c->scheduleGC();
        3687                 :
        3688           84138 :         compartmentCount++;
        3689           84138 :         if (c->isGCScheduled())
        3690           83796 :             collectedCount++;
        3691                 :     }
        3692                 :
        3693           76962 :     gcstats::AutoGCSlice agc(rt->gcStats, collectedCount, compartmentCount, reason);
        3694                 :
        3695           38481 :     do {
        3696                 :         /*
        3697                 :          * Let the API user decide to defer a GC if it wants to (unless this
        3698                 :          * is the last context). Invoke the callback regardless.
        3699                 :          */
        3700           38481 :         if (rt->gcIncrementalState == NO_INCREMENTAL) {
        3701           76854 :             gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_GC_BEGIN);
        3702           38427 :             if (JSGCCallback callback = rt->gcCallback)
        3703               0 :                 callback(rt, JSGC_BEGIN);
        3704                 :         }
        3705                 :
        3706                 :         {
        3707                 :             /* Lock out other GC allocator and collector invocations. */
        3708           76962 :             AutoLockGC lock(rt);
        3709           38481 :             rt->gcPoke = false;
        3710           38481 :             GCCycle(cx, incremental, budget, gckind);
        3711                 :         }
        3712                 :
        3713           38481 :         if (rt->gcIncrementalState == NO_INCREMENTAL) {
        3714           76854 :             gcstats::AutoPhase ap(rt->gcStats, gcstats::PHASE_GC_END);
        3715           38427 :             if (JSGCCallback callback = rt->gcCallback)
        3716               0 :                 callback(rt, JSGC_END);
        3717                 :         }
        3718                 :
        3719                 :         /*
        3720                 :          * On shutdown, iterate until finalizers or the JSGC_END callback
        3721                 :          * stop creating garbage.
        3722                 :          */
        3723           38481 :     } while (!rt->hasContexts() && rt->gcPoke);
        3724                 : }
        3725                 :
        3726                 : namespace js {
        3727                 :
        3728                 : void
        3729           38240 : GC(JSContext *cx, JSGCInvocationKind gckind, gcreason::Reason reason)
        3730                 : {
        3731           38240 :     Collect(cx, false, SliceBudget::Unlimited, gckind, reason);
        3732           38240 : }
        3733                 :
        3734                 : void
        3735             160 : GCSlice(JSContext *cx, JSGCInvocationKind gckind, gcreason::Reason reason)
        3736                 : {
        3737             160 :     Collect(cx, true, cx->runtime->gcSliceBudget, gckind, reason);
        3738             160 : }
        3739                 :
        3740                 : void
        3741              81 : GCDebugSlice(JSContext *cx, bool limit, int64_t objCount)
        3742                 : {
        3743              81 :     int64_t budget = limit ? SliceBudget::WorkBudget(objCount) : SliceBudget::Unlimited;
        3744              81 :     PrepareForDebugGC(cx->runtime);
        3745              81 :     Collect(cx, true, budget, GC_NORMAL, gcreason::API);
        3746              81 : }
        3747                 :
        3748                 : /* Schedule a full GC unless a compartment will already be collected. */
        3749                 : void
        3750           10862 : PrepareForDebugGC(JSRuntime *rt)
        3751                 : {
        3752           34239 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3753           23530 :         if (c->isGCScheduled())
        3754             153 :             return;
        3755                 :     }
        3756                 :
        3757           10709 :     PrepareForFullGC(rt);
        3758                 : }
        3759                 :
        3760                 : void
        3761               9 : ShrinkGCBuffers(JSRuntime *rt)
        3762                 : {
        3763              18 :     AutoLockGC lock(rt);
        3764               9 :     JS_ASSERT(!rt->gcRunning);
        3765                 : #ifndef JS_THREADSAFE
        3766                 :     ExpireChunksAndArenas(rt, true);
        3767                 : #else
        3768               9 :     rt->gcHelperThread.startBackgroundShrink();
        3769                 : #endif
        3770               9 : }
        3771                 :
        3772                 : void
        3773             549 : TraceRuntime(JSTracer *trc)
        3774                 : {
        3775             549 :     JS_ASSERT(!IS_GC_MARKING_TRACER(trc));
        3776                 :
        3777                 : #ifdef JS_THREADSAFE
        3778                 :     {
        3779             549 :         JSRuntime *rt = trc->runtime;
        3780             549 :         if (!rt->gcRunning) {
        3781            1098 :             AutoLockGC lock(rt);
        3782            1098 :             AutoHeapSession session(rt);
        3783                 :
        3784             549 :             rt->gcHelperThread.waitBackgroundSweepEnd();
        3785            1098 :             AutoUnlockGC unlock(rt);
        3786                 :
        3787            1098 :             AutoCopyFreeListToArenas copy(rt);
        3788             549 :             RecordNativeStackTopForGC(rt);
        3789             549 :             MarkRuntime(trc);
        3790                 :             return;
        3791                 :         }
        3792                 :     }
        3793                 : #else
        3794                 :     AutoCopyFreeListToArenas copy(trc->runtime);
        3795                 :     RecordNativeStackTopForGC(trc->runtime);
        3796                 : #endif
        3797                 :
        3798                 :     /*
        3799                 :      * Calls from inside a normal GC or a recursive calls are OK and do not
        3800                 :      * require session setup.
        3801                 :      */
        3802               0 :     MarkRuntime(trc);
        3803                 : }
        3804                 :
        3805                 : struct IterateArenaCallbackOp
        3806                 : {
        3807                 :     JSRuntime *rt;
        3808                 :     void *data;
        3809                 :     IterateArenaCallback callback;
        3810                 :     JSGCTraceKind traceKind;
        3811                 :     size_t thingSize;
        3812               0 :     IterateArenaCallbackOp(JSRuntime *rt, void *data, IterateArenaCallback callback,
        3813                 :                            JSGCTraceKind traceKind, size_t thingSize)
        3814               0 :         : rt(rt), data(data), callback(callback), traceKind(traceKind), thingSize(thingSize)
        3815               0 :     {}
        3816               0 :     void operator()(Arena *arena) { (*callback)(rt, data, arena, traceKind, thingSize); }
        3817                 : };
        3818                 :
        3819                 : struct IterateCellCallbackOp
        3820                 : {
        3821                 :     JSRuntime *rt;
        3822                 :     void *data;
        3823                 :     IterateCellCallback callback;
        3824                 :     JSGCTraceKind traceKind;
        3825                 :     size_t thingSize;
        3826               0 :     IterateCellCallbackOp(JSRuntime *rt, void *data, IterateCellCallback callback,
        3827                 :                           JSGCTraceKind traceKind, size_t thingSize)
        3828               0 :         : rt(rt), data(data), callback(callback), traceKind(traceKind), thingSize(thingSize)
        3829               0 :     {}
        3830               0 :     void operator()(Cell *cell) { (*callback)(rt, data, cell, traceKind, thingSize); }
        3831                 : };
        3832                 :
        3833                 : void
        3834               0 : IterateCompartmentsArenasCells(JSRuntime *rt, void *data,
        3835                 :                                JSIterateCompartmentCallback compartmentCallback,
        3836                 :                                IterateArenaCallback arenaCallback,
        3837                 :                                IterateCellCallback cellCallback)
        3838                 : {
        3839               0 :     JS_ASSERT(!rt->gcRunning);
        3840                 :
        3841               0 :     AutoLockGC lock(rt);
        3842               0 :     AutoHeapSession session(rt);
        3843                 : #ifdef JS_THREADSAFE
        3844               0 :     rt->gcHelperThread.waitBackgroundSweepEnd();
        3845                 : #endif
        3846               0 :     AutoUnlockGC unlock(rt);
        3847                 :
        3848               0 :     AutoCopyFreeListToArenas copy(rt);
        3849               0 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3850               0 :         (*compartmentCallback)(rt, data, c);
        3851                 :
        3852               0 :         for (size_t thingKind = 0; thingKind != FINALIZE_LIMIT; thingKind++) {
        3853               0 :             JSGCTraceKind traceKind = MapAllocToTraceKind(AllocKind(thingKind));
        3854               0 :             size_t thingSize = Arena::thingSize(AllocKind(thingKind));
        3855               0 :             IterateArenaCallbackOp arenaOp(rt, data, arenaCallback, traceKind, thingSize);
        3856               0 :             IterateCellCallbackOp cellOp(rt, data, cellCallback, traceKind, thingSize);
        3857               0 :             ForEachArenaAndCell(c, AllocKind(thingKind), arenaOp, cellOp);
        3858                 :         }
        3859                 :     }
        3860               0 : }
        3861                 :
        3862                 : void
        3863               0 : IterateChunks(JSRuntime *rt, void *data, IterateChunkCallback chunkCallback)
        3864                 : {
        3865                 :     /* :XXX: Any way to common this preamble with IterateCompartmentsArenasCells? */
        3866               0 :     JS_ASSERT(!rt->gcRunning);
        3867                 :
        3868               0 :     AutoLockGC lock(rt);
        3869               0 :     AutoHeapSession session(rt);
        3870                 : #ifdef JS_THREADSAFE
        3871               0 :     rt->gcHelperThread.waitBackgroundSweepEnd();
        3872                 : #endif
        3873               0 :     AutoUnlockGC unlock(rt);
        3874                 :
        3875               0 :     for (js::GCChunkSet::Range r = rt->gcChunkSet.all(); !r.empty(); r.popFront())
        3876               0 :         chunkCallback(rt, data, r.front());
        3877               0 : }
        3878                 :
        3879                 : void
        3880               0 : IterateCells(JSRuntime *rt, JSCompartment *compartment, AllocKind thingKind,
        3881                 :              void *data, IterateCellCallback cellCallback)
        3882                 : {
        3883                 :     /* :XXX: Any way to common this preamble with IterateCompartmentsArenasCells? */
        3884               0 :     JS_ASSERT(!rt->gcRunning);
        3885                 :
        3886               0 :     AutoLockGC lock(rt);
        3887               0 :     AutoHeapSession session(rt);
        3888                 : #ifdef JS_THREADSAFE
        3889               0 :     rt->gcHelperThread.waitBackgroundSweepEnd();
        3890                 : #endif
        3891               0 :     AutoUnlockGC unlock(rt);
        3892                 :
        3893               0 :     AutoCopyFreeListToArenas copy(rt);
        3894                 :
        3895               0 :     JSGCTraceKind traceKind = MapAllocToTraceKind(thingKind);
        3896               0 :     size_t thingSize = Arena::thingSize(thingKind);
        3897                 :
        3898               0 :     if (compartment) {
        3899               0 :         for (CellIterUnderGC i(compartment, thingKind); !i.done(); i.next())
        3900               0 :             cellCallback(rt, data, i.getCell(), traceKind, thingSize);
        3901                 :     } else {
        3902               0 :         for (CompartmentsIter c(rt); !c.done(); c.next()) {
        3903               0 :             for (CellIterUnderGC i(c, thingKind); !i.done(); i.next())
        3904               0 :                 cellCallback(rt, data, i.getCell(), traceKind, thingSize);
        3905                 :         }
        3906                 :     }
        3907               0 : }
        3908                 :
        3909                 : namespace gc {
        3910                 :
        3911                 : JSCompartment *
        3912           22524 : NewCompartment(JSContext *cx, JSPrincipals *principals)
        3913                 : {
        3914           22524 :     JSRuntime *rt = cx->runtime;
        3915           22524 :     JS_AbortIfWrongThread(rt);
        3916                 :
        3917           22524 :     JSCompartment *compartment = cx->new_<JSCompartment>(rt);
        3918           22524 :     if (compartment && compartment->init(cx)) {
        3919                 :         // Any compartment with the trusted principals -- and there can be
        3920                 :         // multiple -- is a system compartment.
        3921           22524 :         compartment->isSystemCompartment = principals && rt->trustedPrincipals() == principals;
        3922           22524 :         if (principals) {
        3923               3 :             compartment->principals = principals;
        3924               3 :             JS_HoldPrincipals(principals);
        3925                 :         }
        3926                 :
        3927           22524 :         compartment->setGCLastBytes(8192, 8192, GC_NORMAL);
        3928                 :
        3929                 :         /*
        3930                 :          * Before reporting the OOM condition, |lock| needs to be cleaned up,
        3931                 :          * hence the scoping.
        3932                 :          */
        3933                 :         {
        3934           45048 :             AutoLockGC lock(rt);
        3935           22524 :             if (rt->compartments.append(compartment))
        3936           22524 :                 return compartment;
        3937                 :         }
        3938                 :
        3939               0 :         js_ReportOutOfMemory(cx);
        3940                 :     }
        3941               0 :     Foreground::delete_(compartment);
        3942               0 :     return NULL;
        3943                 : }
        3944                 :
        3945                 : void
        3946           10664 : RunDebugGC(JSContext *cx)
        3947                 : {
        3948                 : #ifdef JS_GC_ZEAL
        3949           10664 :     PrepareForDebugGC(cx->runtime);
        3950           10664 :     RunLastDitchGC(cx, gcreason::DEBUG_GC);
        3951                 : #endif
        3952           10664 : }
        3953                 :
        3954                 : void
        3955               0 : SetDeterministicGC(JSContext *cx, bool enabled)
        3956                 : {
        3957                 : #ifdef JS_GC_ZEAL
        3958               0 :     JSRuntime *rt = cx->runtime;
        3959               0 :     rt->gcDeterministicOnly = enabled;
        3960                 : #endif
        3961               0 : }
        3962                 :
        3963                 : #if defined(DEBUG) && defined(JSGC_ROOT_ANALYSIS) && !defined(JS_THREADSAFE)
        3964                 :
        3965                 : static void
        3966                 : CheckStackRoot(JSTracer *trc, uintptr_t *w)
        3967                 : {
        3968                 :     /* Mark memory as defined for valgrind, as in MarkWordConservatively. */
        3969                 : #ifdef JS_VALGRIND
        3970                 :     VALGRIND_MAKE_MEM_DEFINED(&w, sizeof(w));
        3971                 : #endif
        3972                 :
        3973                 :     ConservativeGCTest test = MarkIfGCThingWord(trc, *w, DONT_MARK_THING);
        3974                 :
        3975                 :     if (test == CGCT_VALID) {
        3976                 :         JSContext *iter = NULL;
        3977                 :         bool matched = false;
        3978                 :         JSRuntime *rt = trc->runtime;
        3979                 :         for (unsigned i = 0; i < THING_ROOT_COUNT; i++) {
        3980                 :             Root<Cell*> *rooter = rt->thingGCRooters[i];
        3981                 :             while (rooter) {
        3982                 :                 if (rooter->address() == (Cell **) w)
        3983                 :                     matched = true;
        3984                 :                 rooter = rooter->previous();
        3985                 :             }
        3986                 :         }
        3987                 :         CheckRoot *check = rt->checkGCRooters;
        3988                 :         while (check) {
        3989                 :             if (check->contains(static_cast<uint8_t*>(w), sizeof(w)))
        3990                 :                 matched = true;
        3991                 :             check = check->previous();
        3992                 :         }
        3993                 :         if (!matched) {
        3994                 :             /*
        3995                 :              * Only poison the last byte in the word. It is easy to get
        3996                 :              * accidental collisions when a value that does not occupy a full
        3997                 :              * word is used to overwrite a now-dead GC thing pointer. In this
        3998                 :              * case we want to avoid damaging the smaller value.
        3999                 :              */
        4000                 :             PoisonPtr(w);
        4001                 :         }
        4002                 :     }
        4003                 : }
        4004                 :
        4005                 : static void
        4006                 : CheckStackRootsRange(JSTracer *trc, uintptr_t *begin, uintptr_t *end)
        4007                 : {
        4008                 :     JS_ASSERT(begin <= end);
        4009                 :     for (uintptr_t *i = begin; i != end; ++i)
        4010                 :         CheckStackRoot(trc, i);
        4011                 : }
        4012                 :
        4013                 : void
        4014                 : CheckStackRoots(JSContext *cx)
        4015                 : {
        4016                 :     AutoCopyFreeListToArenas copy(cx->runtime);
        4017                 :
        4018                 :     JSTracer checker;
        4019                 :     JS_TracerInit(&checker, cx, EmptyMarkCallback);
        4020                 :
        4021                 :     ThreadData *td = JS_THREAD_DATA(cx);
        4022                 :
        4023                 :     ConservativeGCThreadData *ctd = &td->conservativeGC;
        4024                 :     ctd->recordStackTop();
        4025                 :
        4026                 :     JS_ASSERT(ctd->hasStackToScan());
        4027                 :     uintptr_t *stackMin, *stackEnd;
        4028                 : #if JS_STACK_GROWTH_DIRECTION > 0
        4029                 :     stackMin = td->nativeStackBase;
        4030                 :     stackEnd = ctd->nativeStackTop;
        4031                 : #else
        4032                 :     stackMin = ctd->nativeStackTop + 1;
        4033                 :     stackEnd = td->nativeStackBase;
        4034                 : #endif
        4035                 :
        4036                 :     JS_ASSERT(stackMin <= stackEnd);
        4037                 :     CheckStackRootsRange(&checker, stackMin, stackEnd);
        4038                 :     CheckStackRootsRange(&checker, ctd->registerSnapshot.words,
        4039                 :                          ArrayEnd(ctd->registerSnapshot.words));
        4040                 : }
        4041                 :
        4042                 : #endif /* DEBUG && JSGC_ROOT_ANALYSIS && !JS_THREADSAFE */
        4043                 :
        4044                 : #ifdef JS_GC_ZEAL
        4045                 :
        4046                 : /*
        4047                 :  * Write barrier verification
        4048                 :  *
        4049                 :  * The next few functions are for incremental write barrier verification. When
        4050                 :  * StartVerifyBarriers is called, a snapshot is taken of all objects in the GC
        4051                 :  * heap and saved in an explicit graph data structure. Later, EndVerifyBarriers
        4052                 :  * traverses the heap again. Any pointer values that were in the snapshot and
        4053                 :  * are no longer found must be marked; otherwise an assertion triggers. Note
        4054                 :  * that we must not GC in between starting and finishing a verification phase.
        4055                 :  *
        4056                 :  * The VerifyBarriers function is a shorthand. It checks if a verification phase
        4057                 :  * is currently running. If not, it starts one. Otherwise, it ends the current
        4058                 :  * phase and starts a new one.
        4059                 :  *
        4060                 :  * The user can adjust the frequency of verifications, which causes
        4061                 :  * VerifyBarriers to be a no-op all but one out of N calls. However, if the
        4062                 :  * |always| parameter is true, it starts a new phase no matter what.
        4063                 :  */
        4064                 :
        4065                 : struct EdgeValue
        4066                 : {
        4067                 :     void *thing;
        4068                 :     JSGCTraceKind kind;
        4069                 :     char *label;
        4070                 : };
        4071                 :
        4072                 : struct VerifyNode
        4073                 : {
        4074                 :     void *thing;
        4075                 :     JSGCTraceKind kind;
        4076                 :     uint32_t count;
        4077                 :     EdgeValue edges[1];
        4078                 : };
        4079                 :
        4080                 : typedef HashMap<void *, VerifyNode *, DefaultHasher<void *>, SystemAllocPolicy> NodeMap;
        4081                 :
        4082                 : /*
        4083                 :  * The verifier data structures are simple. The entire graph is stored in a
        4084                 :  * single block of memory. At the beginning is a VerifyNode for the root
        4085                 :  * node. It is followed by a sequence of EdgeValues--the exact number is given
        4086                 :  * in the node. After the edges come more nodes and their edges.
        4087                 :  *
        4088                 :  * The edgeptr and term fields are used to allocate out of the block of memory
        4089                 :  * for the graph. If we run out of memory (i.e., if edgeptr goes beyond term),
        4090                 :  * we just abandon the verification.
        4091                 :  *
        4092                 :  * The nodemap field is a hashtable that maps from the address of the GC thing
        4093                 :  * to the VerifyNode that represents it.
        4094                 :  */
        4095                 : struct VerifyTracer : JSTracer {
        4096                 :     /* The gcNumber when the verification began. */
        4097                 :     uint64_t number;
        4098                 :
        4099                 :     /* This counts up to JS_VERIFIER_FREQ to decide whether to verify. */
        4100                 :     uint32_t count;
        4101                 :
        4102                 :     /* This graph represents the initial GC "snapshot". */
        4103                 :     VerifyNode *curnode;
        4104                 :     VerifyNode *root;
        4105                 :     char *edgeptr;
        4106                 :     char *term;
        4107                 :     NodeMap nodemap;
        4108                 :
        4109            1510 :     VerifyTracer() : root(NULL) {}
        4110            1510 :     ~VerifyTracer() { js_free(root); }
        4111                 : };
        4112                 :
        4113                 : /*
        4114                 :  * This function builds up the heap snapshot by adding edges to the current
        4115                 :  * node.
        4116                 :  */
        4117                 : static void
        4118        18958972 : AccumulateEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind)
        4119                 : {
        4120        18958972 :     VerifyTracer *trc = (VerifyTracer *)jstrc;
        4121                 :
        4122        18958972 :     trc->edgeptr += sizeof(EdgeValue);
        4123        18958972 :     if (trc->edgeptr >= trc->term) {
        4124               0 :         trc->edgeptr = trc->term;
        4125               0 :         return;
        4126                 :     }
        4127                 :
        4128        18958972 :     VerifyNode *node = trc->curnode;
        4129        18958972 :     uint32_t i = node->count;
        4130                 :
        4131        18958972 :     node->edges[i].thing = *thingp;
        4132        18958972 :     node->edges[i].kind = kind;
        4133        18958972 :     node->edges[i].label = trc->debugPrinter ? NULL : (char *)trc->debugPrintArg;
        4134        18958972 :     node->count++;
        4135                 : }
        4136                 :
        4137                 : static VerifyNode *
        4138        18960482 : MakeNode(VerifyTracer *trc, void *thing, JSGCTraceKind kind)
        4139                 : {
        4140        37920964 :     NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing);
        4141        18960482 :     if (!p) {
        4142        11519472 :         VerifyNode *node = (VerifyNode *)trc->edgeptr;
        4143        11519472 :         trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue);
        4144        11519472 :         if (trc->edgeptr >= trc->term) {
        4145               0 :             trc->edgeptr = trc->term;
        4146               0 :             return NULL;
        4147                 :         }
        4148                 :
        4149        11519472 :         node->thing = thing;
        4150        11519472 :         node->count = 0;
        4151        11519472 :         node->kind = kind;
        4152        11519472 :         trc->nodemap.add(p, thing, node);
        4153        11519472 :         return node;
        4154                 :     }
        4155         7441010 :     return NULL;
        4156                 : }
        4157                 :
        4158                 : static
        4159                 : VerifyNode *
        4160        21774610 : NextNode(VerifyNode *node)
        4161                 : {
        4162        21774610 :     if (node->count == 0)
        4163        14613271 :         return (VerifyNode *)((char *)node + sizeof(VerifyNode) - sizeof(EdgeValue));
        4164                 :     else
        4165                 :         return (VerifyNode *)((char *)node + sizeof(VerifyNode) +
        4166         7161339 :                               sizeof(EdgeValue)*(node->count - 1));
        4167                 : }
        4168                 :
        4169                 : static void
        4170            1640 : StartVerifyBarriers(JSContext *cx)
        4171                 : {
        4172            1640 :     JSRuntime *rt = cx->runtime;
        4173                 :
        4174            1640 :     if (rt->gcVerifyData || rt->gcIncrementalState != NO_INCREMENTAL)
        4175               0 :         return;
        4176                 :
        4177            3280 :     AutoLockGC lock(rt);
        4178            3280 :     AutoHeapSession session(rt);
        4179                 :
        4180            1640 :     if (!IsIncrementalGCSafe(rt))
        4181                 :         return;
        4182                 :
        4183                 : #ifdef JS_THREADSAFE
        4184            1510 :     rt->gcHelperThread.waitBackgroundSweepOrAllocEnd();
        4185                 : #endif
        4186                 :
        4187            3020 :     AutoUnlockGC unlock(rt);
        4188                 :
        4189            3020 :     AutoCopyFreeListToArenas copy(rt);
        4190            1510 :     RecordNativeStackTopForGC(rt);
        4191                 :
        4192            4430 :     for (GCChunkSet::Range r(rt->gcChunkSet.all()); !r.empty(); r.popFront())
        4193            2920 :         r.front()->bitmap.clear();
        4194                 :
        4195            6810 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        4196            5300 :         c->discardJitCode(rt->defaultFreeOp());
        4197                 :
        4198            1510 :     VerifyTracer *trc = new (js_malloc(sizeof(VerifyTracer))) VerifyTracer;
        4199                 :
        4200            1510 :     rt->gcNumber++;
        4201            1510 :     trc->number = rt->gcNumber;
        4202            1510 :     trc->count = 0;
        4203                 :
        4204            1510 :     JS_TracerInit(trc, rt, AccumulateEdge);
        4205                 :
        4206            1510 :     PurgeRuntime(trc);
        4207                 :
        4208            1510 :     const size_t size = 64 * 1024 * 1024;
        4209            1510 :     trc->root = (VerifyNode *)js_malloc(size);
        4210            1510 :     JS_ASSERT(trc->root);
        4211            1510 :     trc->edgeptr = (char *)trc->root;
        4212            1510 :     trc->term = trc->edgeptr + size;
        4213                 :
        4214            1510 :     trc->nodemap.init();
        4215                 :
        4216                 :     /* Create the root node. */
        4217            1510 :     trc->curnode = MakeNode(trc, NULL, JSGCTraceKind(0));
        4218                 :
        4219                 :     /* We want MarkRuntime to save the roots to gcSavedRoots. */
        4220            1510 :     rt->gcIncrementalState = MARK_ROOTS;
        4221                 :
        4222                 :     /* Make all the roots be edges emanating from the root node. */
        4223            1510 :     MarkRuntime(trc);
        4224                 :
        4225            1510 :     VerifyNode *node = trc->curnode;
        4226            1510 :     if (trc->edgeptr == trc->term)
        4227               0 :         goto oom;
        4228                 :
        4229                 :     /* For each edge, make a node for it if one doesn't already exist. */
        4230        11522492 :     while ((char *)node < trc->edgeptr) {
        4231        30478444 :         for (uint32_t i = 0; i < node->count; i++) {
        4232        18958972 :             EdgeValue &e = node->edges[i];
        4233        18958972 :             VerifyNode *child = MakeNode(trc, e.thing, e.kind);
        4234        18958972 :             if (child) {
        4235        11517962 :                 trc->curnode = child;
        4236        11517962 :                 JS_TraceChildren(trc, e.thing, e.kind);
        4237                 :             }
        4238        18958972 :             if (trc->edgeptr == trc->term)
        4239               0 :                 goto oom;
        4240                 :         }
        4241                 :
        4242        11519472 :         node = NextNode(node);
        4243                 :     }
        4244                 :
        4245            1510 :     rt->gcVerifyData = trc;
        4246            1510 :     rt->gcIncrementalState = MARK;
        4247            1510 :     rt->gcMarker.start(rt);
        4248            6810 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        4249            5300 :         c->needsBarrier_ = true;
        4250            5300 :         c->arenas.prepareForIncrementalGC(rt);
        4251                 :     }
        4252                 :
        4253                 :     return;
        4254                 :
        4255                 : oom:
        4256               0 :     rt->gcIncrementalState = NO_INCREMENTAL;
        4257               0 :     trc->~VerifyTracer();
        4258            1510 :     js_free(trc);
        4259                 : }
        4260                 :
        4261                 : static void
        4262             762 : MarkFromAutorooter(JSTracer *jstrc, void **thingp, JSGCTraceKind kind)
        4263                 : {
        4264             762 :     static_cast<Cell *>(*thingp)->markIfUnmarked();
        4265             762 : }
        4266                 :
        4267                 : static bool
        4268            2109 : IsMarkedOrAllocated(Cell *cell)
        4269                 : {
        4270            2109 :     return cell->isMarked() || cell->arenaHeader()->allocatedDuringIncremental;
        4271                 : }
        4272                 :
        4273                 : const static uint32_t MAX_VERIFIER_EDGES = 1000;
        4274                 :
        4275                 : /*
        4276                 :  * This function is called by EndVerifyBarriers for every heap edge. If the edge
        4277                 :  * already existed in the original snapshot, we "cancel it out" by overwriting
        4278                 :  * it with NULL. EndVerifyBarriers later asserts that the remaining non-NULL
        4279                 :  * edges (i.e., the ones from the original snapshot that must have been
        4280                 :  * modified) must point to marked objects.
        4281                 :  */
        4282                 : static void
        4283        10812890 : CheckEdge(JSTracer *jstrc, void **thingp, JSGCTraceKind kind)
        4284                 : {
        4285        10812890 :     VerifyTracer *trc = (VerifyTracer *)jstrc;
        4286        10812890 :     VerifyNode *node = trc->curnode;
        4287                 :
        4288                 :     /* Avoid n^2 behavior. */
        4289        10812890 :     if (node->count > MAX_VERIFIER_EDGES)
        4290               0 :         return;
        4291                 :
        4292        50434420 :     for (uint32_t i = 0; i < node->count; i++) {
        4293        50432869 :         if (node->edges[i].thing == *thingp) {
        4294        10811339 :             JS_ASSERT(node->edges[i].kind == kind);
        4295        10811339 :             node->edges[i].thing = NULL;
        4296        10811339 :             return;
        4297                 :         }
        4298                 :     }
        4299                 :
        4300                 :     /*
        4301                 :      * Anything that is reachable now should have been reachable before, or else
        4302                 :      * it should be marked.
        4303                 :      */
        4304            1551 :     NodeMap::Ptr p = trc->nodemap.lookup(*thingp);
        4305            1551 :     JS_ASSERT_IF(!p, IsMarkedOrAllocated(static_cast<Cell *>(*thingp)));
        4306                 : }
        4307                 :
        4308                 : static void
        4309         6273212 : CheckReachable(JSTracer *jstrc, void **thingp, JSGCTraceKind kind)
        4310                 : {
        4311         6273212 :     VerifyTracer *trc = (VerifyTracer *)jstrc;
        4312         6273212 :     NodeMap::Ptr p = trc->nodemap.lookup(*thingp);
        4313         6273212 :     JS_ASSERT_IF(!p, IsMarkedOrAllocated(static_cast<Cell *>(*thingp)));
        4314         6273212 : }
        4315                 :
        4316                 : static void
        4317            1410 : EndVerifyBarriers(JSContext *cx)
        4318                 : {
        4319            1410 :     JSRuntime *rt = cx->runtime;
        4320                 :
        4321            2820 :     AutoLockGC lock(rt);
        4322            2820 :     AutoHeapSession session(rt);
        4323                 :
        4324                 : #ifdef JS_THREADSAFE
        4325            1410 :     rt->gcHelperThread.waitBackgroundSweepOrAllocEnd();
        4326                 : #endif
        4327                 :
        4328            2820 :     AutoUnlockGC unlock(rt);
        4329                 :
        4330            2820 :     AutoCopyFreeListToArenas copy(rt);
        4331            1410 :     RecordNativeStackTopForGC(rt);
        4332                 :
        4333            1410 :     VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData;
        4334                 :
        4335            1410 :     if (!trc)
        4336                 :         return;
        4337                 :
        4338            1410 :     bool compartmentCreated = false;
        4339                 :
        4340                 :     /* We need to disable barriers before tracing, which may invoke barriers. */
        4341            6691 :     for (CompartmentsIter c(rt); !c.done(); c.next()) {
        4342            5281 :         if (!c->needsBarrier_)
        4343              81 :             compartmentCreated = true;
        4344                 :
        4345            5281 :         c->needsBarrier_ = false;
        4346                 :     }
        4347                 :
        4348                 :     /*
        4349                 :      * We need to bump gcNumber so that the methodjit knows that jitcode has
        4350                 :      * been discarded.
        4351                 :      */
        4352            1410 :     JS_ASSERT(trc->number == rt->gcNumber);
        4353            1410 :     rt->gcNumber++;
        4354                 :
        4355            6691 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        4356            5281 :         c->discardJitCode(rt->defaultFreeOp());
        4357                 :
        4358            1410 :     rt->gcVerifyData = NULL;
        4359            1410 :     rt->gcIncrementalState = NO_INCREMENTAL;
        4360                 :
        4361            1410 :     JS_TracerInit(trc, rt, MarkFromAutorooter);
        4362                 :
        4363            1410 :     AutoGCRooter::traceAll(trc);
        4364                 :
        4365            1410 :     if (!compartmentCreated && IsIncrementalGCSafe(rt)) {
        4366                 :         /*
        4367                 :          * Verify that all the current roots were reachable previously, or else
        4368                 :          * are marked.
        4369                 :          */
        4370            1314 :         JS_TracerInit(trc, rt, CheckReachable);
        4371            1314 :         MarkRuntime(trc, true);
        4372                 :
        4373            1314 :         JS_TracerInit(trc, rt, CheckEdge);
        4374                 :
        4375                 :         /* Start after the roots. */
        4376            1314 :         VerifyNode *node = NextNode(trc->root);
        4377        10256452 :         while ((char *)node < trc->edgeptr) {
        4378        10253824 :             trc->curnode = node;
        4379        10253824 :             JS_TraceChildren(trc, node->thing, node->kind);
        4380                 :
        4381        10253824 :             if (node->count <= MAX_VERIFIER_EDGES) {
        4382        21065957 :                 for (uint32_t i = 0; i < node->count; i++) {
        4383        10812133 :                     void *thing = node->edges[i].thing;
        4384        10812133 :                     JS_ASSERT_IF(thing, IsMarkedOrAllocated(static_cast<Cell *>(thing)));
        4385                 :                 }
        4386                 :             }
        4387                 :
        4388        10253824 :             node = NextNode(node);
        4389                 :         }
        4390                 :     }
        4391                 :
        4392            1410 :     rt->gcMarker.reset();
        4393            1410 :     rt->gcMarker.stop();
        4394                 :
        4395            1410 :     trc->~VerifyTracer();
        4396            2820 :     js_free(trc);
        4397                 : }
        4398                 :
        4399                 : void
        4400           18761 : FinishVerifier(JSRuntime *rt)
        4401                 : {
        4402           18761 :     if (VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData) {
        4403             100 :         trc->~VerifyTracer();
        4404             100 :         js_free(trc);
        4405                 :     }
        4406           18761 : }
        4407                 :
        4408                 : void
        4409              18 : VerifyBarriers(JSContext *cx)
        4410                 : {
        4411              18 :     JSRuntime *rt = cx->runtime;
        4412              18 :     if (rt->gcVerifyData)
        4413               0 :         EndVerifyBarriers(cx);
        4414                 :     else
        4415              18 :         StartVerifyBarriers(cx);
        4416              18 : }
        4417                 :
        4418                 : void
        4419     -2051281982 : MaybeVerifyBarriers(JSContext *cx, bool always)
        4420                 : {
        4421     -2051281982 :     if (cx->runtime->gcZeal() != ZealVerifierValue) {
        4422     -2051290904 :         if (cx->runtime->gcVerifyData)
        4423              18 :             EndVerifyBarriers(cx);
        4424     -2051290904 :         return;
        4425                 :     }
        4426                 :
        4427            8922 :     uint32_t freq = cx->runtime->gcZealFrequency;
        4428                 :
        4429            8922 :     JSRuntime *rt = cx->runtime;
        4430            8922 :     if (VerifyTracer *trc = (VerifyTracer *)rt->gcVerifyData) {
        4431            8692 :         if (++trc->count < freq && !always)
        4432            7430 :             return;
        4433                 :
        4434            1262 :         EndVerifyBarriers(cx);
        4435                 :     }
        4436            1492 :     StartVerifyBarriers(cx);
        4437                 : }
        4438                 :
        4439                 : #endif /* JS_GC_ZEAL */
        4440                 :
        4441                 : } /* namespace gc */
        4442                 :
        4443               0 : static void ReleaseAllJITCode(JSContext *cx)
        4444                 : {
        4445                 : #ifdef JS_METHODJIT
        4446               0 :     for (GCCompartmentsIter c(cx->runtime); !c.done(); c.next()) {
        4447               0 :         mjit::ClearAllFrames(c);
        4448               0 :         for (CellIter i(c, FINALIZE_SCRIPT); !i.done(); i.next()) {
        4449               0 :             JSScript *script = i.get<JSScript>();
        4450               0 :             mjit::ReleaseScriptCode(cx->runtime->defaultFreeOp(), script);
        4451                 :         }
        4452                 :     }
        4453                 : #endif
        4454               0 : }
        4455                 :
        4456                 : /*
        4457                 :  * There are three possible PCCount profiling states:
        4458                 :  *
        4459                 :  * 1. None: Neither scripts nor the runtime have count information.
        4460                 :  * 2. Profile: Active scripts have count information, the runtime does not.
        4461                 :  * 3. Query: Scripts do not have count information, the runtime does.
        4462                 :  *
        4463                 :  * When starting to profile scripts, counting begins immediately, with all JIT
        4464                 :  * code discarded and recompiled with counts as necessary. Active interpreter
        4465                 :  * frames will not begin profiling until they begin executing another script
        4466                 :  * (via a call or return).
        4467                 :  *
        4468                 :  * The below API functions manage transitions to new states, according
        4469                 :  * to the table below.
        4470                 :  *
        4471                 :  *                                  Old State
        4472                 :  *                          -------------------------
        4473                 :  * Function                 None      Profile   Query
        4474                 :  * --------
        4475                 :  * StartPCCountProfiling    Profile   Profile   Profile
        4476                 :  * StopPCCountProfiling     None      Query     Query
        4477                 :  * PurgePCCounts            None      None      None
        4478                 :  */
        4479                 :
        4480                 : static void
        4481               0 : ReleaseScriptCounts(JSContext *cx)
        4482                 : {
        4483               0 :     JSRuntime *rt = cx->runtime;
        4484               0 :     JS_ASSERT(rt->scriptAndCountsVector);
        4485                 :
        4486               0 :     ScriptAndCountsVector &vec = *rt->scriptAndCountsVector;
        4487                 :
        4488               0 :     for (size_t i = 0; i < vec.length(); i++)
        4489               0 :         vec[i].scriptCounts.destroy(cx);
        4490                 :
        4491               0 :     cx->delete_(rt->scriptAndCountsVector);
        4492               0 :     rt->scriptAndCountsVector = NULL;
        4493               0 : }
        4494                 :
        4495                 : JS_FRIEND_API(void)
        4496               0 : StartPCCountProfiling(JSContext *cx)
        4497                 : {
        4498               0 :     JSRuntime *rt = cx->runtime;
        4499                 :
        4500               0 :     if (rt->profilingScripts)
        4501               0 :         return;
        4502                 :
        4503               0 :     if (rt->scriptAndCountsVector)
        4504               0 :         ReleaseScriptCounts(cx);
        4505                 :
        4506               0 :     ReleaseAllJITCode(cx);
        4507                 :
        4508               0 :     rt->profilingScripts = true;
        4509                 : }
        4510                 :
        4511                 : JS_FRIEND_API(void)
        4512               0 : StopPCCountProfiling(JSContext *cx)
        4513                 : {
        4514               0 :     JSRuntime *rt = cx->runtime;
        4515                 :
        4516               0 :     if (!rt->profilingScripts)
        4517               0 :         return;
        4518               0 :     JS_ASSERT(!rt->scriptAndCountsVector);
        4519                 :
        4520               0 :     ReleaseAllJITCode(cx);
        4521                 :
        4522               0 :     ScriptAndCountsVector *vec = cx->new_<ScriptAndCountsVector>(SystemAllocPolicy());
        4523               0 :     if (!vec)
        4524               0 :         return;
        4525                 :
        4526               0 :     for (GCCompartmentsIter c(rt); !c.done(); c.next()) {
        4527               0 :         for (CellIter i(c, FINALIZE_SCRIPT); !i.done(); i.next()) {
        4528               0 :             JSScript *script = i.get<JSScript>();
        4529               0 :             if (script->scriptCounts && script->types) {
        4530               0 :                 ScriptAndCounts info;
        4531               0 :                 info.script = script;
        4532               0 :                 info.scriptCounts.steal(script->scriptCounts);
        4533               0 :                 if (!vec->append(info))
        4534               0 :                     info.scriptCounts.destroy(cx);
        4535                 :             }
        4536                 :         }
        4537                 :     }
        4538                 :
        4539               0 :     rt->profilingScripts = false;
        4540               0 :     rt->scriptAndCountsVector = vec;
        4541                 : }
        4542                 :
        4543                 : JS_FRIEND_API(void)
        4544               0 : PurgePCCounts(JSContext *cx)
        4545                 : {
        4546               0 :     JSRuntime *rt = cx->runtime;
        4547                 :
        4548               0 :     if (!rt->scriptAndCountsVector)
        4549               0 :         return;
        4550               0 :     JS_ASSERT(!rt->profilingScripts);
        4551                 :
        4552               0 :     ReleaseScriptCounts(cx);
        4553                 : }
        4554                 :
        4555                 : } /* namespace js */
        4556                 :
        4557                 : JS_PUBLIC_API(void)
        4558               0 : JS_IterateCompartments(JSRuntime *rt, void *data,
        4559                 :                        JSIterateCompartmentCallback compartmentCallback)
        4560                 : {
        4561               0 :     JS_ASSERT(!rt->gcRunning);
        4562                 :
        4563               0 :     AutoLockGC lock(rt);
        4564               0 :     AutoHeapSession session(rt);
        4565                 : #ifdef JS_THREADSAFE
        4566               0 :     rt->gcHelperThread.waitBackgroundSweepOrAllocEnd();
        4567                 : #endif
        4568               0 :     AutoUnlockGC unlock(rt);
        4569                 :
        4570               0 :     for (CompartmentsIter c(rt); !c.done(); c.next())
        4571               0 :         (*compartmentCallback)(rt, data, c);
        4572               0 : }
        4573                 :
        4574                 : #if JS_HAS_XML_SUPPORT
        4575                 : extern size_t sE4XObjectsCreated;
        4576                 :
        4577                 : JSXML *
        4578         4729195 : js_NewGCXML(JSContext *cx)
        4579                 : {
        4580         4729195 :     if (!cx->runningWithTrustedPrincipals())
        4581         4729075 :         ++sE4XObjectsCreated;
        4582                 :
        4583         4729195 :     return NewGCThing<JSXML>(cx, js::gc::FINALIZE_XML, sizeof(JSXML));
        4584                 : }
        4585                 : #endif
        4586                 :

Generated by: LCOV version 1.7