LCOV - app.info - js/src/frontend/TokenStream.cpp

LCOV - code coverage report
| Current view: | directory - js/src/frontend - TokenStream.cpp (source / functions) |  | Found | Hit | Coverage |
| Test: | app.info | Lines: | 1174 | 726 | 61.8 %
| Date: | 2012-04-07 | Functions: | 33 | 26 | 78.8 %

           1                 : /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
           2                 :  * vim: set ts=8 sw=4 et tw=99:
           3                 :  *
           4                 :  * ***** BEGIN LICENSE BLOCK *****
           5                 :  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
           6                 :  *
           7                 :  * The contents of this file are subject to the Mozilla Public License Version
           8                 :  * 1.1 (the "License"); you may not use this file except in compliance with
           9                 :  * the License. You may obtain a copy of the License at
          10                 :  * http://www.mozilla.org/MPL/
          11                 :  *
          12                 :  * Software distributed under the License is distributed on an "AS IS" basis,
          13                 :  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
          14                 :  * for the specific language governing rights and limitations under the
          15                 :  * License.
          16                 :  *
          17                 :  * The Original Code is Mozilla Communicator client code, released
          18                 :  * March 31, 1998.
          19                 :  *
          20                 :  * The Initial Developer of the Original Code is
          21                 :  * Netscape Communications Corporation.
          22                 :  * Portions created by the Initial Developer are Copyright (C) 1998
          23                 :  * the Initial Developer. All Rights Reserved.
          24                 :  *
          25                 :  * Contributor(s):
          26                 :  *   Nick Fitzgerald <nfitzgerald@mozilla.com>
          27                 :  *
          28                 :  * Alternatively, the contents of this file may be used under the terms of
          29                 :  * either of the GNU General Public License Version 2 or later (the "GPL"),
          30                 :  * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
          31                 :  * in which case the provisions of the GPL or the LGPL are applicable instead
          32                 :  * of those above. If you wish to allow use of your version of this file only
          33                 :  * under the terms of either the GPL or the LGPL, and not to allow others to
          34                 :  * use your version of this file under the terms of the MPL, indicate your
          35                 :  * decision by deleting the provisions above and replace them with the notice
          36                 :  * and other provisions required by the GPL or the LGPL. If you do not delete
          37                 :  * the provisions above, a recipient may use your version of this file under
          38                 :  * the terms of any one of the MPL, the GPL or the LGPL.
          39                 :  *
          40                 :  * ***** END LICENSE BLOCK ***** */
          41                 :
          42                 : /*
          43                 :  * JS lexical scanner.
          44                 :  */
          45                 : #include <stdio.h>      /* first to avoid trouble on some systems */
          46                 : #include <errno.h>
          47                 : #include <limits.h>
          48                 : #include <math.h>
          49                 : #ifdef HAVE_MEMORY_H
          50                 : #include <memory.h>
          51                 : #endif
          52                 : #include <stdarg.h>
          53                 : #include <stdlib.h>
          54                 : #include <string.h>
          55                 : #include "jstypes.h"
          56                 : #include "jsutil.h"
          57                 : #include "jsprf.h"
          58                 : #include "jsapi.h"
          59                 : #include "jsatom.h"
          60                 : #include "jscntxt.h"
          61                 : #include "jsversion.h"
          62                 : #include "jsexn.h"
          63                 : #include "jsnum.h"
          64                 : #include "jsopcode.h"
          65                 : #include "jsscript.h"
          66                 :
          67                 : #include "frontend/BytecodeEmitter.h"
          68                 : #include "frontend/Parser.h"
          69                 : #include "frontend/TokenStream.h"
          70                 : #include "vm/RegExpObject.h"
          71                 :
          72                 : #include "jsscriptinlines.h"
          73                 :
          74                 : #if JS_HAS_XML_SUPPORT
          75                 : #include "jsxml.h"
          76                 : #endif
          77                 :
          78                 : using namespace js;
          79                 : using namespace js::unicode;
          80                 :
          81                 : #define JS_KEYWORD(keyword, type, op, version) \
          82                 :     const char js_##keyword##_str[] = #keyword;
          83                 : #include "jskeyword.tbl"
          84                 : #undef JS_KEYWORD
          85                 :
          86                 : static const KeywordInfo keywords[] = {
          87                 : #define JS_KEYWORD(keyword, type, op, version) \
          88                 :     {js_##keyword##_str, type, op, version},
          89                 : #include "jskeyword.tbl"
          90                 : #undef JS_KEYWORD
          91                 : };
          92                 :
          93                 : const KeywordInfo *
          94         4200108 : js::FindKeyword(const jschar *s, size_t length)
          95                 : {
          96         4200108 :     JS_ASSERT(length != 0);
          97                 :
          98                 :     register size_t i;
          99                 :     const struct KeywordInfo *kw;
         100                 :     const char *chars;
         101                 :
         102                 : #define JSKW_LENGTH()           length
         103                 : #define JSKW_AT(column)         s[column]
         104                 : #define JSKW_GOT_MATCH(index)   i = (index); goto got_match;
         105                 : #define JSKW_TEST_GUESS(index)  i = (index); goto test_guess;
         106                 : #define JSKW_NO_MATCH()         goto no_match;
         107                 : #include "jsautokw.h"
         108                 : #undef JSKW_NO_MATCH
         109                 : #undef JSKW_TEST_GUESS
         110                 : #undef JSKW_GOT_MATCH
         111                 : #undef JSKW_AT
         112                 : #undef JSKW_LENGTH
         113                 :
         114                 :   got_match:
         115          719764 :     return &keywords[i];
         116                 :
         117                 :   test_guess:
         118          224019 :     kw = &keywords[i];
         119          224019 :     chars = kw->chars;
         120         1584720 :     do {
         121         1602949 :         if (*s++ != (unsigned char)(*chars++))
         122           18229 :             goto no_match;
         123                 :     } while (--length != 0);
         124          205790 :     return kw;
         125                 :
         126                 :   no_match:
         127         3274554 :     return NULL;
         128                 : }
         129                 :
         130                 : JSBool
         131          462888 : js::IsIdentifier(JSLinearString *str)
         132                 : {
         133          462888 :     const jschar *chars = str->chars();
         134          462888 :     size_t length = str->length();
         135                 :
         136          462888 :     if (length == 0)
         137              27 :         return JS_FALSE;
         138          462861 :     jschar c = *chars;
         139          462861 :     if (!IsIdentifierStart(c))
         140              72 :         return JS_FALSE;
         141          462789 :     const jschar *end = chars + length;
         142         2642427 :     while (++chars != end) {
         143         1716867 :         c = *chars;
         144         1716867 :         if (!IsIdentifierPart(c))
         145              18 :             return JS_FALSE;
         146                 :     }
         147          462771 :     return JS_TRUE;
         148                 : }
         149                 :
         150                 : #ifdef _MSC_VER
         151                 : #pragma warning(push)
         152                 : #pragma warning(disable:4351)
         153                 : #endif
         154                 :
         155                 : /* Initialize members that aren't initialized in |init|. */
         156          114101 : TokenStream::TokenStream(JSContext *cx, JSPrincipals *prin, JSPrincipals *originPrin)
         157                 :   : tokens(), cursor(), lookahead(), flags(), listenerTSData(), tokenbuf(cx),
         158          114101 :     cx(cx), originPrincipals(JSScript::normalizeOriginPrincipals(prin, originPrin))
         159                 : {
         160          114101 :     if (originPrincipals)
         161              69 :         JS_HoldPrincipals(originPrincipals);
         162          114101 : }
         163                 :
         164                 : #ifdef _MSC_VER
         165                 : #pragma warning(pop)
         166                 : #endif
         167                 :
         168                 : bool
         169          114101 : TokenStream::init(const jschar *base, size_t length, const char *fn, unsigned ln, JSVersion v)
         170                 : {
         171          114101 :     filename = fn;
         172          114101 :     lineno = ln;
         173          114101 :     version = v;
         174          114101 :     xml = VersionHasXML(v);
         175                 :
         176          114101 :     userbuf.init(base, length);
         177          114101 :     linebase = base;
         178          114101 :     prevLinebase = NULL;
         179          114101 :     sourceMap = NULL;
         180                 :
         181          114101 :     JSSourceHandler listener = cx->runtime->debugHooks.sourceHandler;
         182          114101 :     void *listenerData = cx->runtime->debugHooks.sourceHandlerData;
         183                 :
         184          114101 :     if (listener)
         185               0 :         listener(fn, ln, base, length, &listenerTSData, listenerData);
         186                 :
         187                 :     /*
         188                 :      * This table holds all the token kinds that satisfy these properties:
         189                 :      * - A single char long.
         190                 :      * - Cannot be a prefix of any longer token (eg. '+' is excluded because
         191                 :      *   '+=' is a valid token).
         192                 :      * - Doesn't need tp->t_op set (eg. this excludes '~').
         193                 :      *
         194                 :      * The few token kinds satisfying these properties cover roughly 35--45%
         195                 :      * of the tokens seen in practice.
         196                 :      *
         197                 :      * Nb: oneCharTokens, maybeEOL and maybeStrSpecial could be static, but
         198                 :      * initializing them this way is a bit easier.  Don't worry, the time to
         199                 :      * initialize them for each TokenStream is trivial.  See bug 639420.
         200                 :      */
         201          114101 :     memset(oneCharTokens, 0, sizeof(oneCharTokens));
         202          114101 :     oneCharTokens[unsigned(';')] = TOK_SEMI;
         203          114101 :     oneCharTokens[unsigned(',')] = TOK_COMMA;
         204          114101 :     oneCharTokens[unsigned('?')] = TOK_HOOK;
         205          114101 :     oneCharTokens[unsigned('[')] = TOK_LB;
         206          114101 :     oneCharTokens[unsigned(']')] = TOK_RB;
         207          114101 :     oneCharTokens[unsigned('{')] = TOK_LC;
         208          114101 :     oneCharTokens[unsigned('}')] = TOK_RC;
         209          114101 :     oneCharTokens[unsigned('(')] = TOK_LP;
         210          114101 :     oneCharTokens[unsigned(')')] = TOK_RP;
         211                 :
         212                 :     /* See getChar() for an explanation of maybeEOL[]. */
         213          114101 :     memset(maybeEOL, 0, sizeof(maybeEOL));
         214          114101 :     maybeEOL[unsigned('\n')] = true;
         215          114101 :     maybeEOL[unsigned('\r')] = true;
         216          114101 :     maybeEOL[unsigned(LINE_SEPARATOR & 0xff)] = true;
         217          114101 :     maybeEOL[unsigned(PARA_SEPARATOR & 0xff)] = true;
         218                 :
         219                 :     /* See getTokenInternal() for an explanation of maybeStrSpecial[]. */
         220          114101 :     memset(maybeStrSpecial, 0, sizeof(maybeStrSpecial));
         221          114101 :     maybeStrSpecial[unsigned('"')] = true;
         222          114101 :     maybeStrSpecial[unsigned('\'')] = true;
         223          114101 :     maybeStrSpecial[unsigned('\\')] = true;
         224          114101 :     maybeStrSpecial[unsigned('\n')] = true;
         225          114101 :     maybeStrSpecial[unsigned('\r')] = true;
         226          114101 :     maybeStrSpecial[unsigned(LINE_SEPARATOR & 0xff)] = true;
         227          114101 :     maybeStrSpecial[unsigned(PARA_SEPARATOR & 0xff)] = true;
         228          114101 :     maybeStrSpecial[unsigned(EOF & 0xff)] = true;
         229                 :
         230                 :     /*
         231                 :      * Set |ln| as the beginning line number of the ungot "current token", so
         232                 :      * that js::Parser::statements (and potentially other such methods, in the
         233                 :      * future) can create parse nodes with good source coordinates before they
         234                 :      * explicitly get any tokens.
         235                 :      *
         236                 :      * Switching the parser/lexer so we always get the next token ahead of the
         237                 :      * parser needing it (the so-called "pump-priming" model) might be a better
         238                 :      * way to address the dependency from statements on the current token.
         239                 :      */
         240          114101 :     tokens[0].pos.begin.lineno = tokens[0].pos.end.lineno = ln;
         241          114101 :     return true;
         242                 : }
         243                 :
         244          228202 : TokenStream::~TokenStream()
         245                 : {
         246          114101 :     if (flags & TSF_OWNFILENAME)
         247               1 :         cx->free_((void *) filename);
         248          114101 :     if (sourceMap)
         249               0 :         cx->free_(sourceMap);
         250          114101 :     if (originPrincipals)
         251              69 :         JS_DropPrincipals(cx->runtime, originPrincipals);
         252          114101 : }
         253                 :
         254                 : /* Use the fastest available getc. */
         255                 : #if defined(HAVE_GETC_UNLOCKED)
         256                 : # define fast_getc getc_unlocked
         257                 : #elif defined(HAVE__GETC_NOLOCK)
         258                 : # define fast_getc _getc_nolock
         259                 : #else
         260                 : # define fast_getc getc
         261                 : #endif
         262                 :
         263                 : JS_ALWAYS_INLINE void
         264         1151523 : TokenStream::updateLineInfoForEOL()
         265                 : {
         266         1151523 :     prevLinebase = linebase;
         267         1151523 :     linebase = userbuf.addressOfNextRawChar();
         268         1151523 :     lineno++;
         269         1151523 : }
         270                 :
         271                 : JS_ALWAYS_INLINE void
         272         1086736 : TokenStream::updateFlagsForEOL()
         273                 : {
         274         1086736 :     flags &= ~TSF_DIRTYLINE;
         275         1086736 :     flags |= TSF_EOL;
         276         1086736 : }
         277                 :
         278                 : /* This gets the next char, normalizing all EOL sequences to '\n' as it goes. */
         279                 : int32_t
         280         7693042 : TokenStream::getChar()
         281                 : {
         282                 :     int32_t c;
         283         7693042 :     if (JS_LIKELY(userbuf.hasRawChars())) {
         284         7692970 :         c = userbuf.getRawChar();
         285                 :
         286                 :         /*
         287                 :          * Normalize the jschar if it was a newline.  We need to detect any of
         288                 :          * these four characters:  '\n' (0x000a), '\r' (0x000d),
         289                 :          * LINE_SEPARATOR (0x2028), PARA_SEPARATOR (0x2029).  Testing for each
         290                 :          * one in turn is slow, so we use a single probabilistic check, and if
         291                 :          * that succeeds, test for them individually.
         292                 :          *
         293                 :          * We use the bottom 8 bits to index into a lookup table, succeeding
         294                 :          * when d&0xff is 0xa, 0xd, 0x28 or 0x29.  Among ASCII chars (which
         295                 :          * are by the far the most common) this gives false positives for '('
         296                 :          * (0x0028) and ')' (0x0029).  We could avoid those by incorporating
         297                 :          * the 13th bit of d into the lookup, but that requires extra shifting
         298                 :          * and masking and isn't worthwhile.  See TokenStream::init() for the
         299                 :          * initialization of the relevant entries in the table.
         300                 :          */
         301         7692970 :         if (JS_UNLIKELY(maybeEOL[c & 0xff])) {
         302          179935 :             if (c == '\n')
         303           67784 :                 goto eol;
         304          112151 :             if (c == '\r') {
         305                 :                 /* if it's a \r\n sequence: treat as a single EOL, skip over the \n */
         306             369 :                 if (userbuf.hasRawChars())
         307             369 :                     userbuf.matchRawChar('\n');
         308             369 :                 goto eol;
         309                 :             }
         310          111782 :             if (c == LINE_SEPARATOR || c == PARA_SEPARATOR)
         311                 :                 goto eol;
         312                 :         }
         313         7624817 :         return c;
         314                 :     }
         315                 :
         316              72 :     flags |= TSF_EOF;
         317              72 :     return EOF;
         318                 :
         319                 :   eol:
         320           68153 :     updateLineInfoForEOL();
         321           68153 :     return '\n';
         322                 : }
         323                 :
         324                 : /*
         325                 :  * This gets the next char. It does nothing special with EOL sequences, not
         326                 :  * even updating the line counters.  It can be used safely if (a) the
         327                 :  * resulting char is guaranteed to be ungotten (by ungetCharIgnoreEOL()) if
         328                 :  * it's an EOL, and (b) the line-related state (lineno, linebase) is not used
         329                 :  * before it's ungotten.
         330                 :  */
         331                 : int32_t
         332        24958702 : TokenStream::getCharIgnoreEOL()
         333                 : {
         334        24958702 :     if (JS_LIKELY(userbuf.hasRawChars()))
         335        24945330 :         return userbuf.getRawChar();
         336                 :
         337           13372 :     flags |= TSF_EOF;
         338           13372 :     return EOF;
         339                 : }
         340                 :
         341                 : void
         342         4081024 : TokenStream::ungetChar(int32_t c)
         343                 : {
         344         4081024 :     if (c == EOF)
         345              71 :         return;
         346         4080953 :     JS_ASSERT(!userbuf.atStart());
         347         4080953 :     userbuf.ungetRawChar();
         348         4080953 :     if (c == '\n') {
         349                 : #ifdef DEBUG
         350           36005 :         int32_t c2 = userbuf.peekRawChar();
         351           36005 :         JS_ASSERT(TokenBuf::isRawEOLChar(c2));
         352                 : #endif
         353                 :
         354                 :         /* if it's a \r\n sequence, also unget the \r */
         355           36005 :         if (!userbuf.atStart())
         356           36005 :             userbuf.matchRawCharBackwards('\r');
         357                 :
         358           36005 :         JS_ASSERT(prevLinebase);    /* we should never get more than one EOL char */
         359           36005 :         linebase = prevLinebase;
         360           36005 :         prevLinebase = NULL;
         361           36005 :         lineno--;
         362                 :     } else {
         363         4044948 :         JS_ASSERT(userbuf.peekRawChar() == c);
         364                 :     }
         365                 : }
         366                 :
         367                 : void
         368         5397975 : TokenStream::ungetCharIgnoreEOL(int32_t c)
         369                 : {
         370         5397975 :     if (c == EOF)
         371           13354 :         return;
         372         5384621 :     JS_ASSERT(!userbuf.atStart());
         373         5384621 :     userbuf.ungetRawChar();
         374                 : }
         375                 :
         376                 : /*
         377                 :  * Return true iff |n| raw characters can be read from this without reading past
         378                 :  * EOF or a newline, and copy those characters into |cp| if so.  The characters
         379                 :  * are not consumed: use skipChars(n) to do so after checking that the consumed
         380                 :  * characters had appropriate values.
         381                 :  */
         382                 : bool
         383           36591 : TokenStream::peekChars(int n, jschar *cp)
         384                 : {
         385                 :     int i, j;
         386                 :     int32_t c;
         387                 :
         388          492908 :     for (i = 0; i < n; i++) {
         389          461369 :         c = getCharIgnoreEOL();
         390          461369 :         if (c == EOF)
         391              18 :             break;
         392          461351 :         if (c == '\n') {
         393            5034 :             ungetCharIgnoreEOL(c);
         394            5034 :             break;
         395                 :         }
         396          456317 :         cp[i] = (jschar)c;
         397                 :     }
         398          492908 :     for (j = i - 1; j >= 0; j--)
         399          456317 :         ungetCharIgnoreEOL(cp[j]);
         400           36591 :     return i == n;
         401                 : }
         402                 :
         403                 : const jschar *
         404             598 : TokenStream::TokenBuf::findEOL()
         405                 : {
         406             598 :     const jschar *tmp = ptr;
         407                 : #ifdef DEBUG
         408                 :     /*
         409                 :      * This is the one exception to the "TokenBuf isn't accessed after
         410                 :      * poisoning" rule -- we may end up calling findEOL() in order to set up
         411                 :      * an error.
         412                 :      */
         413             598 :     if (!tmp)
         414               0 :         tmp = ptrWhenPoisoned;
         415                 : #endif
         416                 :
         417            6138 :     while (true) {
         418            6736 :         if (tmp >= limit)
         419             580 :             break;
         420            6156 :         if (TokenBuf::isRawEOLChar(*tmp++))
         421              18 :             break;
         422                 :     }
         423             598 :     return tmp;
         424                 : }
         425                 :
         426                 : bool
         427            1319 : TokenStream::reportCompileErrorNumberVA(ParseNode *pn, unsigned flags, unsigned errorNumber, va_list ap)
         428                 : {
         429                 :     JSErrorReport report;
         430                 :     char *message;
         431                 :     jschar *linechars;
         432                 :     char *linebytes;
         433                 :     bool warning;
         434                 :     JSBool ok;
         435                 :     const TokenPos *tp;
         436                 :     unsigned i;
         437                 :
         438            1319 :     if (JSREPORT_IS_STRICT(flags) && !cx->hasStrictOption())
         439             712 :         return true;
         440                 :
         441             607 :     warning = JSREPORT_IS_WARNING(flags);
         442             607 :     if (warning && cx->hasWErrorOption()) {
         443               0 :         flags &= ~JSREPORT_WARNING;
         444               0 :         warning = false;
         445                 :     }
         446                 :
         447             607 :     PodZero(&report);
         448             607 :     report.flags = flags;
         449             607 :     report.errorNumber = errorNumber;
         450             607 :     message = NULL;
         451             607 :     linechars = NULL;
         452             607 :     linebytes = NULL;
         453                 :
         454                 :     MUST_FLOW_THROUGH("out");
         455                 :     ok = js_ExpandErrorArguments(cx, js_GetErrorMessage, NULL,
         456                 :                                  errorNumber, &message, &report,
         457             607 :                                  !(flags & JSREPORT_UC), ap);
         458             607 :     if (!ok) {
         459               0 :         warning = false;
         460               0 :         goto out;
         461                 :     }
         462                 :
         463             607 :     report.filename = filename;
         464             607 :     report.originPrincipals = originPrincipals;
         465                 :
         466             607 :     tp = pn ? &pn->pn_pos : &currentToken().pos;
         467             607 :     report.lineno = tp->begin.lineno;
         468                 :
         469                 :     /*
         470                 :      * Given a token, T, that we want to complain about: if T's (starting)
         471                 :      * lineno doesn't match TokenStream's lineno, that means we've scanned past
         472                 :      * the line that T starts on, which makes it hard to print some or all of
         473                 :      * T's (starting) line for context.
         474                 :      *
         475                 :      * So we don't even try, leaving report.linebuf and friends zeroed.  This
         476                 :      * means that any error involving a multi-line token (eg. an unterminated
         477                 :      * multi-line string literal) won't have a context printed.
         478                 :      */
         479             607 :     if (report.lineno == lineno) {
         480             598 :         size_t linelength = userbuf.findEOL() - linebase;
         481                 :
         482             598 :         linechars = (jschar *)cx->malloc_((linelength + 1) * sizeof(jschar));
         483             598 :         if (!linechars) {
         484               0 :             warning = false;
         485               0 :             goto out;
         486                 :         }
         487             598 :         PodCopy(linechars, linebase, linelength);
         488             598 :         linechars[linelength] = 0;
         489             598 :         linebytes = DeflateString(cx, linechars, linelength);
         490             598 :         if (!linebytes) {
         491               0 :             warning = false;
         492               0 :             goto out;
         493                 :         }
         494                 :
         495                 :         /* Unicode and char versions of the offending source line, without final \n */
         496             598 :         report.linebuf = linebytes;
         497             598 :         report.uclinebuf = linechars;
         498                 :
         499                 :         /* The lineno check above means we should only see single-line tokens here. */
         500             598 :         JS_ASSERT(tp->begin.lineno == tp->end.lineno);
         501             598 :         report.tokenptr = report.linebuf + tp->begin.index;
         502             598 :         report.uctokenptr = report.uclinebuf + tp->begin.index;
         503                 :     }
         504                 :
         505                 :     /*
         506                 :      * If there's a runtime exception type associated with this error
         507                 :      * number, set that as the pending exception.  For errors occuring at
         508                 :      * compile time, this is very likely to be a JSEXN_SYNTAXERR.
         509                 :      *
         510                 :      * If an exception is thrown but not caught, the JSREPORT_EXCEPTION
         511                 :      * flag will be set in report.flags.  Proper behavior for an error
         512                 :      * reporter is to ignore a report with this flag for all but top-level
         513                 :      * compilation errors.  The exception will remain pending, and so long
         514                 :      * as the non-top-level "load", "eval", or "compile" native function
         515                 :      * returns false, the top-level reporter will eventually receive the
         516                 :      * uncaught exception report.
         517                 :      */
         518             607 :     if (!js_ErrorToException(cx, message, &report, NULL, NULL)) {
         519                 :         /*
         520                 :          * If debugErrorHook is present then we give it a chance to veto
         521                 :          * sending the error on to the regular error reporter.
         522                 :          */
         523               9 :         bool reportError = true;
         524               9 :         if (JSDebugErrorHook hook = cx->runtime->debugHooks.debugErrorHook)
         525               0 :             reportError = hook(cx, message, &report, cx->runtime->debugHooks.debugErrorHookData);
         526                 :
         527                 :         /* Report the error */
         528               9 :         if (reportError && cx->errorReporter)
         529               9 :             cx->errorReporter(cx, message, &report);
         530                 :     }
         531                 :
         532                 :   out:
         533             607 :     if (linebytes)
         534             598 :         cx->free_(linebytes);
         535             607 :     if (linechars)
         536             598 :         cx->free_(linechars);
         537             607 :     if (message)
         538             607 :         cx->free_(message);
         539             607 :     if (report.ucmessage)
         540             607 :         cx->free_((void *)report.ucmessage);
         541                 :
         542             607 :     if (report.messageArgs) {
         543             207 :         if (!(flags & JSREPORT_UC)) {
         544             207 :             i = 0;
         545             810 :             while (report.messageArgs[i])
         546             396 :                 cx->free_((void *)report.messageArgs[i++]);
         547                 :         }
         548             207 :         cx->free_((void *)report.messageArgs);
         549                 :     }
         550                 :
         551             607 :     return warning;
         552                 : }
         553                 :
         554                 : bool
         555            1485 : js::ReportStrictModeError(JSContext *cx, TokenStream *ts, TreeContext *tc, ParseNode *pn,
         556                 :                           unsigned errorNumber, ...)
         557                 : {
         558            1485 :     JS_ASSERT(ts || tc);
         559            1485 :     JS_ASSERT(cx == ts->getContext());
         560                 :
         561                 :     /* In strict mode code, this is an error, not merely a warning. */
         562                 :     unsigned flags;
         563            1485 :     if ((ts && ts->isStrictMode()) || (tc && (tc->flags & TCF_STRICT_MODE_CODE))) {
         564               0 :         flags = JSREPORT_ERROR;
         565                 :     } else {
         566            1485 :         if (!cx->hasStrictOption())
         567            1485 :             return true;
         568               0 :         flags = JSREPORT_WARNING;
         569                 :     }
         570                 :
         571                 :     va_list ap;
         572               0 :     va_start(ap, errorNumber);
         573               0 :     bool result = ts->reportCompileErrorNumberVA(pn, flags, errorNumber, ap);
         574               0 :     va_end(ap);
         575                 :
         576               0 :     return result;
         577                 : }
         578                 :
         579                 : bool
         580             883 : js::ReportCompileErrorNumber(JSContext *cx, TokenStream *ts, ParseNode *pn, unsigned flags,
         581                 :                              unsigned errorNumber, ...)
         582                 : {
         583                 :     va_list ap;
         584                 :
         585                 :     /*
         586                 :      * We don't accept a TreeContext argument, so we can't implement
         587                 :      * JSREPORT_STRICT_MODE_ERROR here.  Use ReportStrictModeError instead,
         588                 :      * or do the checks in the caller and pass plain old JSREPORT_ERROR.
         589                 :      */
         590             883 :     JS_ASSERT(!(flags & JSREPORT_STRICT_MODE_ERROR));
         591                 :
         592             883 :     va_start(ap, errorNumber);
         593             883 :     JS_ASSERT(cx == ts->getContext());
         594             883 :     bool result = ts->reportCompileErrorNumberVA(pn, flags, errorNumber, ap);
         595             883 :     va_end(ap);
         596                 :
         597             883 :     return result;
         598                 : }
         599                 :
         600                 : #if JS_HAS_XML_SUPPORT
         601                 :
         602                 : bool
         603               0 : TokenStream::getXMLEntity()
         604                 : {
         605                 :     ptrdiff_t offset, length, i;
         606                 :     int c, d;
         607                 :     JSBool ispair;
         608                 :     jschar *bp, digit;
         609                 :     char *bytes;
         610                 :     JSErrNum msg;
         611                 :
         612               0 :     CharBuffer &tb = tokenbuf;
         613                 :
         614                 :     /* Put the entity, including the '&' already scanned, in tokenbuf. */
         615               0 :     offset = tb.length();
         616               0 :     if (!tb.append('&'))
         617               0 :         return false;
         618               0 :     while ((c = getChar()) != ';') {
         619               0 :         if (c == EOF || c == '\n') {
         620               0 :             ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_END_OF_XML_ENTITY);
         621               0 :             return false;
         622                 :         }
         623               0 :         if (!tb.append(c))
         624               0 :             return false;
         625                 :     }
         626                 :
         627                 :     /* Let length be the number of jschars after the '&', including the ';'. */
         628               0 :     length = tb.length() - offset;
         629               0 :     bp = tb.begin() + offset;
         630               0 :     c = d = 0;
         631               0 :     ispair = false;
         632               0 :     if (length > 2 && bp[1] == '#') {
         633                 :         /* Match a well-formed XML Character Reference. */
         634               0 :         i = 2;
         635               0 :         if (length > 3 && (bp[i] == 'x' || bp[i] == 'X')) {
         636               0 :             if (length > 9)     /* at most 6 hex digits allowed */
         637               0 :                 goto badncr;
         638               0 :             while (++i < length) {
         639               0 :                 digit = bp[i];
         640               0 :                 if (!JS7_ISHEX(digit))
         641                 :                     goto badncr;
         642               0 :                 c = (c << 4) + JS7_UNHEX(digit);
         643                 :             }
         644                 :         } else {
         645               0 :             while (i < length) {
         646               0 :                 digit = bp[i++];
         647               0 :                 if (!JS7_ISDEC(digit))
         648               0 :                     goto badncr;
         649               0 :                 c = (c * 10) + JS7_UNDEC(digit);
         650               0 :                 if (c < 0)
         651               0 :                     goto badncr;
         652                 :             }
         653                 :         }
         654                 :
         655               0 :         if (0x10000 <= c && c <= 0x10FFFF) {
         656                 :             /* Form a surrogate pair (c, d) -- c is the high surrogate. */
         657               0 :             d = 0xDC00 + (c & 0x3FF);
         658               0 :             c = 0xD7C0 + (c >> 10);
         659               0 :             ispair = true;
         660                 :         } else {
         661                 :             /* Enforce the http://www.w3.org/TR/REC-xml/#wf-Legalchar WFC. */
         662               0 :             if (c != 0x9 && c != 0xA && c != 0xD &&
         663               0 :                 !(0x20 <= c && c <= 0xD7FF) &&
         664               0 :                 !(0xE000 <= c && c <= 0xFFFD)) {
         665                 :                 goto badncr;
         666                 :             }
         667                 :         }
         668                 :     } else {
         669                 :         /* Try to match one of the five XML 1.0 predefined entities. */
         670               0 :         switch (length) {
         671                 :           case 3:
         672               0 :             if (bp[2] == 't') {
         673               0 :                 if (bp[1] == 'l')
         674               0 :                     c = '<';
         675               0 :                 else if (bp[1] == 'g')
         676               0 :                     c = '>';
         677                 :             }
         678               0 :             break;
         679                 :           case 4:
         680               0 :             if (bp[1] == 'a' && bp[2] == 'm' && bp[3] == 'p')
         681               0 :                 c = '&';
         682               0 :             break;
         683                 :           case 5:
         684               0 :             if (bp[3] == 'o') {
         685               0 :                 if (bp[1] == 'a' && bp[2] == 'p' && bp[4] == 's')
         686               0 :                     c = '\'';
         687               0 :                 else if (bp[1] == 'q' && bp[2] == 'u' && bp[4] == 't')
         688               0 :                     c = '"';
         689                 :             }
         690               0 :             break;
         691                 :         }
         692               0 :         if (c == 0) {
         693               0 :             msg = JSMSG_UNKNOWN_XML_ENTITY;
         694               0 :             goto bad;
         695                 :         }
         696                 :     }
         697                 :
         698                 :     /* If we matched, retract tokenbuf and store the entity's value. */
         699               0 :     *bp++ = (jschar) c;
         700               0 :     if (ispair)
         701               0 :         *bp++ = (jschar) d;
         702               0 :     tb.shrinkBy(tb.end() - bp);
         703               0 :     return true;
         704                 :
         705                 :   badncr:
         706               0 :     msg = JSMSG_BAD_XML_NCR;
         707                 :   bad:
         708                 :     /* No match: throw a TypeError per ECMA-357 10.3.2.1 step 8(a). */
         709               0 :     JS_ASSERT((tb.end() - bp) >= 1);
         710               0 :     bytes = DeflateString(cx, bp + 1, (tb.end() - bp) - 1);
         711               0 :     if (bytes) {
         712               0 :         ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, msg, bytes);
         713               0 :         cx->free_(bytes);
         714                 :     }
         715               0 :     return false;
         716                 : }
         717                 :
         718                 : bool
         719           16850 : TokenStream::getXMLTextOrTag(TokenKind *ttp, Token **tpp)
         720                 : {
         721                 :     TokenKind tt;
         722                 :     int c, qc;
         723                 :     Token *tp;
         724                 :     JSAtom *atom;
         725                 :
         726                 :     /*
         727                 :      * Look for XML text.
         728                 :      */
         729           16850 :     if (flags & TSF_XMLTEXTMODE) {
         730            2632 :         tt = TOK_XMLSPACE;      /* veto if non-space, return TOK_XMLTEXT */
         731            2632 :         tp = newToken(0);
         732            2632 :         tokenbuf.clear();
         733            2632 :         qc = (flags & TSF_XMLONLYMODE) ? '<' : '{';
         734                 :
         735            6227 :         while ((c = getChar()) != qc && c != '<' && c != EOF) {
         736             963 :             if (c == '&' && qc == '<') {
         737               0 :                 if (!getXMLEntity())
         738               0 :                     goto error;
         739               0 :                 tt = TOK_XMLTEXT;
         740               0 :                 continue;
         741                 :             }
         742                 :
         743             963 :             if (!IsXMLSpace(c))
         744             963 :                 tt = TOK_XMLTEXT;
         745             963 :             if (!tokenbuf.append(c))
         746               0 :                 goto error;
         747                 :         }
         748            2632 :         ungetChar(c);
         749                 :
         750            2632 :         if (tokenbuf.empty()) {
         751            1723 :             atom = NULL;
         752                 :         } else {
         753             909 :             atom = atomize(cx, tokenbuf);
         754             909 :             if (!atom)
         755               0 :                 goto error;
         756                 :         }
         757            2632 :         tp->pos.end.lineno = lineno;
         758            2632 :         tp->setAtom(JSOP_STRING, atom);
         759            2632 :         goto out;
         760                 :     }
         761                 :
         762                 :     /*
         763                 :      * XML tags.
         764                 :      */
         765                 :     else {
         766           14218 :         JS_ASSERT(flags & TSF_XMLTAGMODE);
         767           14218 :         tp = newToken(0);
         768           14218 :         c = getChar();
         769           14218 :         if (c != EOF && IsXMLSpace(c)) {
         770            1396 :             do {
         771            1396 :                 c = getChar();
         772            1396 :                 if (c == EOF)
         773               0 :                     break;
         774            1396 :             } while (IsXMLSpace(c));
         775            1396 :             ungetChar(c);
         776            1396 :             tp->pos.end.lineno = lineno;
         777            1396 :             tt = TOK_XMLSPACE;
         778            1396 :             goto out;
         779                 :         }
         780                 :
         781           12822 :         if (c == EOF) {
         782               0 :             tt = TOK_EOF;
         783               0 :             goto out;
         784                 :         }
         785                 :
         786           12822 :         tokenbuf.clear();
         787           12822 :         if (IsXMLNamespaceStart(c)) {
         788            5695 :             JSBool sawColon = JS_FALSE;
         789                 :
         790            5695 :             if (!tokenbuf.append(c))
         791               0 :                 goto error;
         792           32094 :             while ((c = getChar()) != EOF && IsXMLNamePart(c)) {
         793           20704 :                 if (c == ':') {
         794                 :                     int nextc;
         795                 :
         796               0 :                     if (sawColon ||
         797                 :                         (nextc = peekChar(),
         798                 :                          ((flags & TSF_XMLONLYMODE) || nextc != '{') &&
         799               0 :                          !IsXMLNamePart(nextc))) {
         800                 :                         ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
         801               0 :                                                  JSMSG_BAD_XML_QNAME);
         802               0 :                         goto error;
         803                 :                     }
         804               0 :                     sawColon = JS_TRUE;
         805                 :                 }
         806                 :
         807           20704 :                 if (!tokenbuf.append(c))
         808               0 :                     goto error;
         809                 :             }
         810                 :
         811            5695 :             ungetChar(c);
         812            5695 :             atom = atomize(cx, tokenbuf);
         813            5695 :             if (!atom)
         814               0 :                 goto error;
         815            5695 :             tp->setAtom(JSOP_STRING, atom);
         816            5695 :             tt = TOK_XMLNAME;
         817            5695 :             goto out;
         818                 :         }
         819                 :
         820            7127 :         switch (c) {
         821                 :           case '{':
         822               0 :             if (flags & TSF_XMLONLYMODE)
         823               0 :                 goto bad_xml_char;
         824               0 :             tt = TOK_LC;
         825               0 :             goto out;
         826                 :
         827                 :           case '=':
         828            1396 :             tt = TOK_ASSIGN;
         829            1396 :             goto out;
         830                 :
         831                 :           case '"':
         832                 :           case '\'':
         833            1396 :             qc = c;
         834            2810 :             while ((c = getChar()) != qc) {
         835              18 :                 if (c == EOF) {
         836                 :                     ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
         837               0 :                                              JSMSG_UNTERMINATED_STRING);
         838               0 :                     goto error;
         839                 :                 }
         840                 :
         841                 :                 /*
         842                 :                  * XML attribute values are double-quoted when pretty-printed,
         843                 :                  * so escape " if it is expressed directly in a single-quoted
         844                 :                  * attribute value.
         845                 :                  */
         846              18 :                 if (c == '"' && !(flags & TSF_XMLONLYMODE)) {
         847               0 :                     JS_ASSERT(qc == '\'');
         848               0 :                     if (!tokenbuf.append(js_quot_entity_str,
         849               0 :                                      strlen(js_quot_entity_str)))
         850               0 :                         goto error;
         851               0 :                     continue;
         852                 :                 }
         853                 :
         854              18 :                 if (c == '&' && (flags & TSF_XMLONLYMODE)) {
         855               0 :                     if (!getXMLEntity())
         856               0 :                         goto error;
         857               0 :                     continue;
         858                 :                 }
         859                 :
         860              18 :                 if (!tokenbuf.append(c))
         861               0 :                     goto error;
         862                 :             }
         863            1396 :             atom = atomize(cx, tokenbuf);
         864            1396 :             if (!atom)
         865               0 :                 goto error;
         866            1396 :             tp->pos.end.lineno = lineno;
         867            1396 :             tp->setAtom(JSOP_STRING, atom);
         868            1396 :             tt = TOK_XMLATTR;
         869            1396 :             goto out;
         870                 :
         871                 :           case '>':
         872            3858 :             tt = TOK_XMLTAGC;
         873            3858 :             goto out;
         874                 :
         875                 :           case '/':
         876             477 :             if (matchChar('>')) {
         877             477 :                 tt = TOK_XMLPTAGC;
         878             477 :                 goto out;
         879                 :             }
         880                 :             /* FALL THROUGH */
         881                 :
         882                 :           bad_xml_char:
         883                 :           default:
         884               0 :             ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_BAD_XML_CHARACTER);
         885               0 :             goto error;
         886                 :         }
         887                 :         JS_NOT_REACHED("getXMLTextOrTag 1");
         888                 :     }
         889                 :     JS_NOT_REACHED("getXMLTextOrTag 2");
         890                 :
         891                 :   out:
         892           16850 :     *ttp = tt;
         893           16850 :     *tpp = tp;
         894           16850 :     return true;
         895                 :
         896                 :   error:
         897               0 :     *ttp = TOK_ERROR;
         898               0 :     *tpp = tp;
         899               0 :     return false;
         900                 : }
         901                 :
         902                 : /*
         903                 :  * After much testing, it's clear that Postel's advice to protocol designers
         904                 :  * ("be liberal in what you accept, and conservative in what you send") invites
         905                 :  * a natural-law repercussion for JS as "protocol":
         906                 :  *
         907                 :  * "If you are liberal in what you accept, others will utterly fail to be
         908                 :  *  conservative in what they send."
         909                 :  *
         910                 :  * Which means you will get <!-- comments to end of line in the middle of .js
         911                 :  * files, and after if conditions whose then statements are on the next line,
         912                 :  * and other wonders.  See at least the following bugs:
         913                 :  * - https://bugzilla.mozilla.org/show_bug.cgi?id=309242
         914                 :  * - https://bugzilla.mozilla.org/show_bug.cgi?id=309712
         915                 :  * - https://bugzilla.mozilla.org/show_bug.cgi?id=310993
         916                 :  *
         917                 :  * So without JSOPTION_XML, we changed around Firefox 1.5 never to scan an XML
         918                 :  * comment or CDATA literal.  Instead, we always scan <! as the start of an
         919                 :  * HTML comment hack to end of line, used since Netscape 2 to hide script tag
         920                 :  * content from script-unaware browsers.
         921                 :  *
         922                 :  * But this still leaves XML resources with certain internal structure
         923                 :  * vulnerable to being loaded as script cross-origin, and some internal data
         924                 :  * stolen, so for Firefox 3.5 and beyond, we reject programs whose source
         925                 :  * consists only of XML literals. See:
         926                 :  *
         927                 :  * https://bugzilla.mozilla.org/show_bug.cgi?id=336551
         928                 :  *
         929                 :  * The check for this is in js::frontend::CompileScript.
         930                 :  */
         931                 : bool
         932            4362 : TokenStream::getXMLMarkup(TokenKind *ttp, Token **tpp)
         933                 : {
         934                 :     TokenKind tt;
         935                 :     int c;
         936            4362 :     Token *tp = *tpp;
         937                 :
         938                 :     /* Check for XML comment or CDATA section. */
         939            4362 :     if (matchChar('!')) {
         940              27 :         tokenbuf.clear();
         941                 :
         942                 :         /* Scan XML comment. */
         943              27 :         if (matchChar('-')) {
         944               0 :             if (!matchChar('-'))
         945               0 :                 goto bad_xml_markup;
         946               0 :             while ((c = getChar()) != '-' || !matchChar('-')) {
         947               0 :                 if (c == EOF)
         948               0 :                     goto bad_xml_markup;
         949               0 :                 if (!tokenbuf.append(c))
         950               0 :                     goto error;
         951                 :             }
         952               0 :             if (!matchChar('>'))
         953               0 :                 goto bad_xml_markup;
         954                 :
         955               0 :             JSAtom *commentText = atomize(cx, tokenbuf);
         956               0 :             if (!commentText)
         957               0 :                 goto error;
         958               0 :             tp->setAtom(JSOP_XMLCOMMENT, commentText);
         959               0 :             tp->pos.end.lineno = lineno;
         960               0 :             tt = TOK_XMLCOMMENT;
         961               0 :             goto out;
         962                 :         }
         963                 :
         964                 :         /* Scan CDATA section. */
         965              27 :         if (matchChar('[')) {
         966                 :             jschar cp[6];
         967             189 :             if (peekChars(6, cp) &&
         968              27 :                 cp[0] == 'C' &&
         969              27 :                 cp[1] == 'D' &&
         970              27 :                 cp[2] == 'A' &&
         971              27 :                 cp[3] == 'T' &&
         972              27 :                 cp[4] == 'A' &&
         973              27 :                 cp[5] == '[') {
         974              27 :                 skipChars(6);
         975             108 :                 while ((c = getChar()) != ']' ||
         976              27 :                        !peekChars(2, cp) ||
         977              27 :                        cp[0] != ']' ||
         978              27 :                        cp[1] != '>') {
         979             270 :                     if (c == EOF)
         980               0 :                         goto bad_xml_markup;
         981             270 :                     if (!tokenbuf.append(c))
         982               0 :                         goto error;
         983                 :                 }
         984              27 :                 consumeKnownChar(']');
         985              27 :                 consumeKnownChar('>');
         986                 :
         987              27 :                 JSAtom *cdataContent = atomize(cx, tokenbuf);
         988              27 :                 if (!cdataContent)
         989               0 :                     goto error;
         990                 :
         991              27 :                 tp->setAtom(JSOP_XMLCDATA, cdataContent);
         992              27 :                 tp->pos.end.lineno = lineno;
         993              27 :                 tt = TOK_XMLCDATA;
         994              27 :                 goto out;
         995                 :             }
         996               0 :             goto bad_xml_markup;
         997                 :         }
         998                 :     }
         999                 :
        1000                 :     /* Check for processing instruction. */
        1001            4335 :     if (matchChar('?')) {
        1002               0 :         bool inTarget = true;
        1003               0 :         size_t targetLength = 0;
        1004               0 :         ptrdiff_t contentIndex = -1;
        1005                 :
        1006               0 :         tokenbuf.clear();
        1007               0 :         while ((c = getChar()) != '?' || peekChar() != '>') {
        1008               0 :             if (c == EOF)
        1009               0 :                 goto bad_xml_markup;
        1010               0 :             if (inTarget) {
        1011               0 :                 if (IsXMLSpace(c)) {
        1012               0 :                     if (tokenbuf.empty())
        1013               0 :                         goto bad_xml_markup;
        1014               0 :                     inTarget = false;
        1015                 :                 } else {
        1016               0 :                     if (!(tokenbuf.empty()
        1017               0 :                           ? IsXMLNamespaceStart(c)
        1018               0 :                           : IsXMLNamespacePart(c))) {
        1019               0 :                         goto bad_xml_markup;
        1020                 :                     }
        1021               0 :                     ++targetLength;
        1022                 :                 }
        1023                 :             } else {
        1024               0 :                 if (contentIndex < 0 && !IsXMLSpace(c))
        1025               0 :                     contentIndex = tokenbuf.length();
        1026                 :             }
        1027               0 :             if (!tokenbuf.append(c))
        1028               0 :                 goto error;
        1029                 :         }
        1030               0 :         if (targetLength == 0)
        1031               0 :             goto bad_xml_markup;
        1032                 :
        1033                 :         JSAtom *data;
        1034               0 :         if (contentIndex < 0) {
        1035               0 :             data = cx->runtime->atomState.emptyAtom;
        1036                 :         } else {
        1037               0 :             data = js_AtomizeChars(cx, tokenbuf.begin() + contentIndex,
        1038               0 :                                    tokenbuf.length() - contentIndex);
        1039               0 :             if (!data)
        1040               0 :                 goto error;
        1041                 :         }
        1042               0 :         tokenbuf.shrinkBy(tokenbuf.length() - targetLength);
        1043               0 :         consumeKnownChar('>');
        1044               0 :         JSAtom *target = atomize(cx, tokenbuf);
        1045               0 :         if (!target)
        1046               0 :             goto error;
        1047               0 :         tp->setProcessingInstruction(target->asPropertyName(), data);
        1048               0 :         tp->pos.end.lineno = lineno;
        1049               0 :         tt = TOK_XMLPI;
        1050               0 :         goto out;
        1051                 :     }
        1052                 :
        1053                 :     /* An XML start-of-tag character. */
        1054            4335 :     tt = matchChar('/') ? TOK_XMLETAGO : TOK_XMLSTAGO;
        1055                 :
        1056                 :   out:
        1057            4362 :     *ttp = tt;
        1058            4362 :     *tpp = tp;
        1059            4362 :     return true;
        1060                 :
        1061                 :   bad_xml_markup:
        1062               0 :     ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_BAD_XML_MARKUP);
        1063                 :   error:
        1064               0 :     *ttp = TOK_ERROR;
        1065               0 :     *tpp = tp;
        1066               0 :     return false;
        1067                 : }
        1068                 : #endif /* JS_HAS_XML_SUPPORT */
        1069                 :
        1070                 : /*
        1071                 :  * We have encountered a '\': check for a Unicode escape sequence after it.
        1072                 :  * Return 'true' and the character code value (by value) if we found a
        1073                 :  * Unicode escape sequence.  Otherwise, return 'false'.  In both cases, do not
        1074                 :  * advance along the buffer.
        1075                 :  */
        1076                 : bool
        1077               0 : TokenStream::peekUnicodeEscape(int *result)
        1078                 : {
        1079                 :     jschar cp[5];
        1080                 :
        1081               0 :     if (peekChars(5, cp) && cp[0] == 'u' &&
        1082               0 :         JS7_ISHEX(cp[1]) && JS7_ISHEX(cp[2]) &&
        1083               0 :         JS7_ISHEX(cp[3]) && JS7_ISHEX(cp[4]))
        1084                 :     {
        1085               0 :         *result = (((((JS7_UNHEX(cp[1]) << 4)
        1086               0 :                 + JS7_UNHEX(cp[2])) << 4)
        1087               0 :               + JS7_UNHEX(cp[3])) << 4)
        1088               0 :             + JS7_UNHEX(cp[4]);
        1089               0 :         return true;
        1090                 :     }
        1091               0 :     return false;
        1092                 : }
        1093                 :
        1094                 : bool
        1095               0 : TokenStream::matchUnicodeEscapeIdStart(int32_t *cp)
        1096                 : {
        1097               0 :     if (peekUnicodeEscape(cp) && IsIdentifierStart(*cp)) {
        1098               0 :         skipChars(5);
        1099               0 :         return true;
        1100                 :     }
        1101               0 :     return false;
        1102                 : }
        1103                 :
        1104                 : bool
        1105               0 : TokenStream::matchUnicodeEscapeIdent(int32_t *cp)
        1106                 : {
        1107               0 :     if (peekUnicodeEscape(cp) && IsIdentifierPart(*cp)) {
        1108               0 :         skipChars(5);
        1109               0 :         return true;
        1110                 :     }
        1111               0 :     return false;
        1112                 : }
        1113                 :
        1114                 : /*
        1115                 :  * Helper function which returns true if the first length(q) characters in p are
        1116                 :  * the same as the characters in q.
        1117                 :  */
        1118                 : static bool
        1119           19758 : CharsMatch(const jschar *p, const char *q) {
        1120           39544 :     while (*q) {
        1121           19783 :         if (*p++ != *q++)
        1122           19755 :             return false;
        1123                 :     }
        1124               3 :     return true;
        1125                 : }
        1126                 :
        1127                 : bool
        1128               2 : TokenStream::getAtLine()
        1129                 : {
        1130                 :     int c;
        1131                 :     jschar cp[5];
        1132                 :     unsigned i, line, temp;
        1133                 :     char filenameBuf[1024];
        1134                 :
        1135                 :     /*
        1136                 :      * Hack for source filters such as the Mozilla XUL preprocessor:
        1137                 :      * "//@line 123\n" sets the number of the *next* line after the
        1138                 :      * comment to 123.  If we reach here, we've already seen "//".
        1139                 :      */
        1140               2 :     if (peekChars(5, cp) && CharsMatch(cp, "@line")) {
        1141               2 :         skipChars(5);
        1142               6 :         while ((c = getChar()) != '\n' && c != EOF && IsSpaceOrBOM2(c))
        1143               2 :             continue;
        1144               2 :         if (JS7_ISDEC(c)) {
        1145               2 :             line = JS7_UNDEC(c);
        1146               8 :             while ((c = getChar()) != EOF && JS7_ISDEC(c)) {
        1147               4 :                 temp = 10 * line + JS7_UNDEC(c);
        1148               4 :                 if (temp < line) {
        1149                 :                     /* Ignore overlarge line numbers. */
        1150               0 :                     return true;
        1151                 :                 }
        1152               4 :                 line = temp;
        1153                 :             }
        1154               6 :             while (c != '\n' && c != EOF && IsSpaceOrBOM2(c))
        1155               2 :                 c = getChar();
        1156               2 :             i = 0;
        1157               2 :             if (c == '"') {
        1158              10 :                 while ((c = getChar()) != EOF && c != '"') {
        1159               6 :                     if (c == '\n') {
        1160               0 :                         ungetChar(c);
        1161               0 :                         return true;
        1162                 :                     }
        1163               6 :                     if ((c >> 8) != 0 || i >= sizeof filenameBuf - 1)
        1164               0 :                         return true;
        1165               6 :                     filenameBuf[i++] = (char) c;
        1166                 :                 }
        1167               2 :                 if (c == '"') {
        1168               4 :                     while ((c = getChar()) != '\n' && c != EOF && IsSpaceOrBOM2(c))
        1169               0 :                         continue;
        1170                 :                 }
        1171                 :             }
        1172               2 :             filenameBuf[i] = '\0';
        1173               2 :             if (c == EOF || c == '\n') {
        1174               2 :                 if (i > 0) {
        1175               2 :                     if (flags & TSF_OWNFILENAME)
        1176               1 :                         cx->free_((void *) filename);
        1177               2 :                     filename = JS_strdup(cx, filenameBuf);
        1178               2 :                     if (!filename)
        1179               0 :                         return false;
        1180               2 :                     flags |= TSF_OWNFILENAME;
        1181                 :                 }
        1182               2 :                 lineno = line;
        1183                 :             }
        1184                 :         }
        1185               2 :         ungetChar(c);
        1186                 :     }
        1187               2 :     return true;
        1188                 : }
        1189                 :
        1190                 : bool
        1191           24808 : TokenStream::getAtSourceMappingURL()
        1192                 : {
        1193                 :     jschar peeked[18];
        1194                 :
        1195                 :     /* Match comments of the form @sourceMappingURL=<url> */
        1196           24808 :     if (peekChars(18, peeked) && CharsMatch(peeked, "@sourceMappingURL=")) {
        1197               1 :         skipChars(18);
        1198               1 :         tokenbuf.clear();
        1199                 :
        1200                 :         jschar c;
        1201              44 :         while (!IsSpaceOrBOM2((c = getChar())) &&
        1202                 :                c && c != jschar(EOF))
        1203              42 :             tokenbuf.append(c);
        1204                 :
        1205               1 :         if (tokenbuf.empty())
        1206                 :             /* The source map's URL was missing, but not quite an exception that
        1207                 :              * we should stop and drop everything for, though. */
        1208               0 :             return true;
        1209                 :
        1210               1 :         int len = tokenbuf.length();
        1211                 :
        1212               1 :         if (sourceMap)
        1213               0 :             cx->free_(sourceMap);
        1214               1 :         sourceMap = (jschar *) cx->malloc_(sizeof(jschar) * (len + 1));
        1215               1 :         if (!sourceMap)
        1216               0 :             return false;
        1217                 :
        1218              43 :         for (int i = 0; i < len; i++)
        1219              42 :             sourceMap[i] = tokenbuf[i];
        1220               1 :         sourceMap[len] = '\0';
        1221                 :     }
        1222           24808 :     return true;
        1223                 : }
        1224                 :
        1225                 : Token *
        1226        11640550 : TokenStream::newToken(ptrdiff_t adjust)
        1227                 : {
        1228        11640550 :     cursor = (cursor + 1) & ntokensMask;
        1229        11640550 :     Token *tp = &tokens[cursor];
        1230        11640550 :     tp->ptr = userbuf.addressOfNextRawChar() + adjust;
        1231        11640550 :     tp->pos.begin.index = tp->ptr - linebase;
        1232        11640550 :     tp->pos.begin.lineno = tp->pos.end.lineno = lineno;
        1233        11640550 :     return tp;
        1234                 : }
        1235                 :
        1236                 : JS_ALWAYS_INLINE JSAtom *
        1237          305456 : TokenStream::atomize(JSContext *cx, CharBuffer &cb)
        1238                 : {
        1239          305456 :     return js_AtomizeChars(cx, cb.begin(), cb.length());
        1240                 : }
        1241                 :
        1242                 : #ifdef DEBUG
        1243                 : bool
        1244        11589228 : IsTokenSane(Token *tp)
        1245                 : {
        1246                 :     /*
        1247                 :      * Nb: TOK_EOL should never be used in an actual Token;  it should only be
        1248                 :      * returned as a TokenKind from peekTokenSameLine().
        1249                 :      */
        1250        11589228 :     if (tp->type < TOK_ERROR || tp->type >= TOK_LIMIT || tp->type == TOK_EOL)
        1251               0 :         return false;
        1252                 :
        1253        11589228 :     if (tp->pos.begin.lineno == tp->pos.end.lineno) {
        1254        11588598 :         if (tp->pos.begin.index > tp->pos.end.index)
        1255               0 :             return false;
        1256                 :     } else {
        1257                 :         /* Only certain token kinds can be multi-line. */
        1258             630 :         switch (tp->type) {
        1259                 :           case TOK_STRING:
        1260                 :           case TOK_XMLATTR:
        1261                 :           case TOK_XMLSPACE:
        1262                 :           case TOK_XMLTEXT:
        1263                 :           case TOK_XMLCOMMENT:
        1264                 :           case TOK_XMLCDATA:
        1265                 :           case TOK_XMLPI:
        1266             630 :             break;
        1267                 :           default:
        1268               0 :             return false;
        1269                 :         }
        1270                 :     }
        1271        11589228 :     return true;
        1272                 : }
        1273                 : #endif
        1274                 :
        1275                 : bool
        1276               0 : TokenStream::putIdentInTokenbuf(const jschar *identStart)
        1277                 : {
        1278                 :     int32_t c, qc;
        1279               0 :     const jschar *tmp = userbuf.addressOfNextRawChar();
        1280               0 :     userbuf.setAddressOfNextRawChar(identStart);
        1281                 :
        1282               0 :     tokenbuf.clear();
        1283               0 :     for (;;) {
        1284               0 :         c = getCharIgnoreEOL();
        1285               0 :         if (!IsIdentifierPart(c)) {
        1286               0 :             if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
        1287                 :                 break;
        1288               0 :             c = qc;
        1289                 :         }
        1290               0 :         if (!tokenbuf.append(c)) {
        1291               0 :             userbuf.setAddressOfNextRawChar(tmp);
        1292               0 :             return false;
        1293                 :         }
        1294                 :     }
        1295               0 :     userbuf.setAddressOfNextRawChar(tmp);
        1296               0 :     return true;
        1297                 : }
        1298                 :
        1299                 : bool
        1300         4198357 : TokenStream::checkForKeyword(const jschar *s, size_t length, TokenKind *ttp, JSOp *topp)
        1301                 : {
        1302         4198357 :     JS_ASSERT(!ttp == !topp);
        1303                 :
        1304         4198357 :     const KeywordInfo *kw = FindKeyword(s, length);
        1305         4198357 :     if (!kw)
        1306         3272812 :         return true;
        1307                 :
        1308          925545 :     if (kw->tokentype == TOK_RESERVED) {
        1309                 :         return ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1310               0 :                                         JSMSG_RESERVED_ID, kw->chars);
        1311                 :     }
        1312                 :
        1313          925545 :     if (kw->tokentype != TOK_STRICT_RESERVED) {
        1314          925545 :         if (kw->version <= versionNumber()) {
        1315                 :             /* Working keyword. */
        1316          925545 :             if (ttp) {
        1317          925545 :                 *ttp = kw->tokentype;
        1318          925545 :                 *topp = (JSOp) kw->op;
        1319          925545 :                 return true;
        1320                 :             }
        1321                 :             return ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1322               0 :                                             JSMSG_RESERVED_ID, kw->chars);
        1323                 :         }
        1324                 :
        1325                 :         /* The let keyword is reserved on <1.7 */
        1326               0 :         if (kw->tokentype == TOK_LET) {
        1327                 :             return ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1328               0 :                                             JSMSG_RESERVED_ID, kw->chars);
        1329                 :         }
        1330                 :
        1331                 :         /*
        1332                 :          * The keyword is not in this version. Treat it as an identifier,
        1333                 :          * unless it is yield which we treat as TOK_STRICT_RESERVED by
        1334                 :          * falling through to the code below (ES5 forbids it in strict mode).
        1335                 :          */
        1336               0 :         if (kw->tokentype != TOK_YIELD)
        1337               0 :             return true;
        1338                 :     }
        1339                 :
        1340                 :     /* Strict reserved word. */
        1341               0 :     if (isStrictMode())
        1342               0 :         return ReportStrictModeError(cx, this, NULL, NULL, JSMSG_RESERVED_ID, kw->chars);
        1343                 :     return ReportCompileErrorNumber(cx, this, NULL, JSREPORT_STRICT | JSREPORT_WARNING,
        1344               0 :                                     JSMSG_RESERVED_ID, kw->chars);
        1345                 : }
        1346                 :
        1347                 : enum FirstCharKind {
        1348                 :     Other,
        1349                 :     OneChar,
        1350                 :     Ident,
        1351                 :     Dot,
        1352                 :     Equals,
        1353                 :     String,
        1354                 :     Dec,
        1355                 :     Colon,
        1356                 :     Plus,
        1357                 :     HexOct,
        1358                 :
        1359                 :     /* These two must be last, so that |c >= Space| matches both. */
        1360                 :     Space,
        1361                 :     EOL
        1362                 : };
        1363                 :
        1364                 : #define _______ Other
        1365                 :
        1366                 : /*
        1367                 :  * OneChar: 40, 41, 44, 59, 63, 91, 93, 123, 125: '(', ')', ',', ';', '?', '[', ']', '{', '}'
        1368                 :  * Ident:   36, 65..90, 95, 97..122: '$', 'A'..'Z', '_', 'a'..'z'
        1369                 :  * Dot:     46: '.'
        1370                 :  * Equals:  61: '='
        1371                 :  * String:  34, 39: '"', '\''
        1372                 :  * Dec:     49..57: '1'..'9'
        1373                 :  * Colon:   58: ':'
        1374                 :  * Plus:    43: '+'
        1375                 :  * HexOct:  48: '0'
        1376                 :  * Space:   9, 11, 12: '\t', '\v', '\f'
        1377                 :  * EOL:     10, 13: '\n', '\r'
        1378                 :  */
        1379                 : static const uint8_t firstCharKinds[] = {
        1380                 : /*         0        1        2        3        4        5        6        7        8        9    */
        1381                 : /*   0+ */ _______, _______, _______, _______, _______, _______, _______, _______, _______,   Space,
        1382                 : /*  10+ */     EOL,   Space,   Space,     EOL, _______, _______, _______, _______, _______, _______,
        1383                 : /*  20+ */ _______, _______, _______, _______, _______, _______, _______, _______, _______, _______,
        1384                 : /*  30+ */ _______, _______,   Space, _______,  String, _______,   Ident, _______, _______,  String,
        1385                 : /*  40+ */ OneChar, OneChar, _______,    Plus, OneChar, _______,     Dot, _______,  HexOct,     Dec,
        1386                 : /*  50+ */     Dec,     Dec,     Dec,     Dec,     Dec,     Dec,     Dec,     Dec,   Colon, OneChar,
        1387                 : /*  60+ */ _______,  Equals, _______, OneChar, _______,   Ident,   Ident,   Ident,   Ident,   Ident,
        1388                 : /*  70+ */   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,
        1389                 : /*  80+ */   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,
        1390                 : /*  90+ */   Ident, OneChar, _______, OneChar, _______,   Ident, _______,   Ident,   Ident,   Ident,
        1391                 : /* 100+ */   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,
        1392                 : /* 110+ */   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,   Ident,
        1393                 : /* 120+ */   Ident,   Ident,   Ident, OneChar, _______, OneChar, _______, _______
        1394                 : };
        1395                 :
        1396                 : #undef _______
        1397                 :
        1398                 : TokenKind
        1399        11589228 : TokenStream::getTokenInternal()
        1400                 : {
        1401                 :     TokenKind tt;
        1402                 :     int c, qc;
        1403                 :     Token *tp;
        1404                 :     FirstCharKind c1kind;
        1405                 :     const jschar *numStart;
        1406                 :     bool hasFracOrExp;
        1407                 :     const jschar *identStart;
        1408                 :     bool hadUnicodeEscape;
        1409                 :
        1410                 : #if JS_HAS_XML_SUPPORT
        1411                 :     /*
        1412                 :      * Look for XML text and tags.
        1413                 :      */
        1414        11589228 :     if (flags & (TSF_XMLTEXTMODE|TSF_XMLTAGMODE)) {
        1415           16850 :         if (!getXMLTextOrTag(&tt, &tp))
        1416               0 :             goto error;
        1417           16850 :         goto out;
        1418                 :     }
        1419                 : #endif
        1420                 :
        1421                 :   retry:
        1422        20759786 :     if (JS_UNLIKELY(!userbuf.hasRawChars())) {
        1423          163644 :         tp = newToken(0);
        1424          163644 :         tt = TOK_EOF;
        1425          163644 :         flags |= TSF_EOF;
        1426          163644 :         goto out;
        1427                 :     }
        1428                 :
        1429        20596142 :     c = userbuf.getRawChar();
        1430        20596142 :     JS_ASSERT(c != EOF);
        1431                 :
        1432                 :     /*
        1433                 :      * Chars not in the range 0..127 are rare.  Getting them out of the way
        1434                 :      * early allows subsequent checking to be faster.
        1435                 :      */
        1436        20596142 :     if (JS_UNLIKELY(c >= 128)) {
        1437               0 :         if (IsSpaceOrBOM2(c)) {
        1438               0 :             if (c == LINE_SEPARATOR || c == PARA_SEPARATOR) {
        1439               0 :                 updateLineInfoForEOL();
        1440               0 :                 updateFlagsForEOL();
        1441                 :             }
        1442                 :
        1443               0 :             goto retry;
        1444                 :         }
        1445                 :
        1446               0 :         tp = newToken(-1);
        1447                 :
        1448                 :         /* '$' and '_' don't pass IsLetter, but they're < 128 so never appear here. */
        1449                 :         JS_STATIC_ASSERT('$' < 128 && '_' < 128);
        1450               0 :         if (IsLetter(c)) {
        1451               0 :             identStart = userbuf.addressOfNextRawChar() - 1;
        1452               0 :             hadUnicodeEscape = false;
        1453               0 :             goto identifier;
        1454                 :         }
        1455                 :
        1456               0 :         goto badchar;
        1457                 :     }
        1458                 :
        1459                 :     /*
        1460                 :      * Get the token kind, based on the first char.  The ordering of c1kind
        1461                 :      * comparison is based on the frequency of tokens in real code.  Minified
        1462                 :      * and non-minified code have different characteristics, mostly in that
        1463                 :      * whitespace occurs much less in minified code.  Token kinds that fall in
        1464                 :      * the 'Other' category typically account for less than 2% of all tokens,
        1465                 :      * so their order doesn't matter much.
        1466                 :      */
        1467        20596142 :     c1kind = FirstCharKind(firstCharKinds[c]);
        1468                 :
        1469                 :     /*
        1470                 :      * Skip over whitespace chars;  update line state on EOLs.  Even though
        1471                 :      * whitespace isn't very common in minified code we have to handle it first
        1472                 :      * (and jump back to 'retry') before calling newToken().
        1473                 :      */
        1474        20596142 :     if (c1kind >= Space) {
        1475         9136086 :         if (c1kind == EOL) {
        1476                 :             /* If it's a \r\n sequence: treat as a single EOL, skip over the \n. */
        1477         1083370 :             if (c == '\r' && userbuf.hasRawChars())
        1478           11151 :                 userbuf.matchRawChar('\n');
        1479         1083370 :             updateLineInfoForEOL();
        1480         1083370 :             updateFlagsForEOL();
        1481                 :         }
        1482         9136086 :         goto retry;
        1483                 :     }
        1484                 :
        1485        11460056 :     tp = newToken(-1);
        1486                 :
        1487                 :     /*
        1488                 :      * Look for an unambiguous single-char token.
        1489                 :      */
        1490        11460056 :     if (c1kind == OneChar) {
        1491         3817956 :         tt = (TokenKind)oneCharTokens[c];
        1492         3817956 :         goto out;
        1493                 :     }
        1494                 :
        1495                 :     /*
        1496                 :      * Look for an identifier.
        1497                 :      */
        1498         7642100 :     if (c1kind == Ident) {
        1499         4393295 :         identStart = userbuf.addressOfNextRawChar() - 1;
        1500         4393295 :         hadUnicodeEscape = false;
        1501                 :
        1502                 :       identifier:
        1503         9369154 :         for (;;) {
        1504        13762449 :             c = getCharIgnoreEOL();
        1505        13762449 :             if (c == EOF)
        1506           11409 :                 break;
        1507        13751040 :             if (!IsIdentifierPart(c)) {
        1508         4381886 :                 if (c != '\\' || !matchUnicodeEscapeIdent(&qc))
        1509         4381886 :                     break;
        1510               0 :                 hadUnicodeEscape = true;
        1511                 :             }
        1512                 :         }
        1513         4393295 :         ungetCharIgnoreEOL(c);
        1514                 :
        1515                 :         /* Convert the escapes by putting into tokenbuf. */
        1516         4393295 :         if (hadUnicodeEscape && !putIdentInTokenbuf(identStart))
        1517               0 :             goto error;
        1518                 :
        1519                 :         /* Check for keywords unless parser asks us to ignore keywords. */
        1520         4393295 :         if (!(flags & TSF_KEYWORD_IS_NAME)) {
        1521                 :             const jschar *chars;
        1522                 :             size_t length;
        1523         4198159 :             if (hadUnicodeEscape) {
        1524               0 :                 chars = tokenbuf.begin();
        1525               0 :                 length = tokenbuf.length();
        1526                 :             } else {
        1527         4198159 :                 chars = identStart;
        1528         4198159 :                 length = userbuf.addressOfNextRawChar() - identStart;
        1529                 :             }
        1530         4198159 :             tt = TOK_NAME;
        1531         4198159 :             if (!checkForKeyword(chars, length, &tt, &tp->t_op))
        1532               0 :                 goto error;
        1533         4198159 :             if (tt != TOK_NAME)
        1534          925545 :                 goto out;
        1535                 :         }
        1536                 :
        1537                 :         /*
        1538                 :          * Identifiers containing no Unicode escapes can be atomized directly
        1539                 :          * from userbuf.  The rest must use the escapes converted via
        1540                 :          * tokenbuf before atomizing.
        1541                 :          */
        1542                 :         JSAtom *atom;
        1543         3467750 :         if (!hadUnicodeEscape)
        1544         3467750 :             atom = js_AtomizeChars(cx, identStart, userbuf.addressOfNextRawChar() - identStart);
        1545                 :         else
        1546               0 :             atom = atomize(cx, tokenbuf);
        1547         3467750 :         if (!atom)
        1548               0 :             goto error;
        1549         3467750 :         tp->setName(JSOP_NAME, atom->asPropertyName());
        1550         3467750 :         tt = TOK_NAME;
        1551         3467750 :         goto out;
        1552                 :     }
        1553                 :
        1554         3248805 :     if (c1kind == Dot) {
        1555          143399 :         c = getCharIgnoreEOL();
        1556          143399 :         if (JS7_ISDEC(c)) {
        1557             180 :             numStart = userbuf.addressOfNextRawChar() - 2;
        1558             180 :             goto decimal_dot;
        1559                 :         }
        1560                 : #if JS_HAS_XML_SUPPORT
        1561          143219 :         if (c == '.') {
        1562               9 :             tt = TOK_DBLDOT;
        1563               9 :             goto out;
        1564                 :         }
        1565                 : #endif
        1566          143210 :         ungetCharIgnoreEOL(c);
        1567          143210 :         tt = TOK_DOT;
        1568          143210 :         goto out;
        1569                 :     }
        1570                 :
        1571         3105406 :     if (c1kind == Equals) {
        1572          545258 :         if (matchChar('=')) {
        1573           14277 :             if (matchChar('=')) {
        1574            9533 :                 tp->t_op = JSOP_STRICTEQ;
        1575            9533 :                 tt = TOK_STRICTEQ;
        1576                 :             } else {
        1577            4744 :                 tp->t_op = JSOP_EQ;
        1578            4744 :                 tt = TOK_EQ;
        1579                 :             }
        1580                 :         } else {
        1581          530981 :             tp->t_op = JSOP_NOP;
        1582          530981 :             tt = TOK_ASSIGN;
        1583                 :         }
        1584          545258 :         goto out;
        1585                 :     }
        1586                 :
        1587                 :     /*
        1588                 :      * Look for a string.
        1589                 :      */
        1590         2560148 :     if (c1kind == String) {
        1591          297429 :         qc = c;
        1592          297429 :         tokenbuf.clear();
        1593         9739504 :         while (true) {
        1594                 :             /*
        1595                 :              * We need to detect any of these chars:  " or ', \n (or its
        1596                 :              * equivalents), \\, EOF.  We use maybeStrSpecial[] in a manner
        1597                 :              * similar to maybeEOL[], see above.  Because we detect EOL
        1598                 :              * sequences here and put them back immediately, we can use
        1599                 :              * getCharIgnoreEOL().
        1600                 :              */
        1601        10036933 :             c = getCharIgnoreEOL();
        1602        10036933 :             if (maybeStrSpecial[c & 0xff]) {
        1603          895306 :                 if (c == qc)
        1604                 :                     break;
        1605          597877 :                 if (c == '\\') {
        1606          258948 :                     switch (c = getChar()) {
        1607              36 :                       case 'b': c = '\b'; break;
        1608              54 :                       case 'f': c = '\f'; break;
        1609          124083 :                       case 'n': c = '\n'; break;
        1610              63 :                       case 'r': c = '\r'; break;
        1611              63 :                       case 't': c = '\t'; break;
        1612              36 :                       case 'v': c = '\v'; break;
        1613                 :
        1614                 :                       default:
        1615          134613 :                         if ('0' <= c && c < '8') {
        1616             738 :                             int32_t val = JS7_UNDEC(c);
        1617                 :
        1618             738 :                             c = peekChar();
        1619                 :                             /* Strict mode code allows only \0, then a non-digit. */
        1620             738 :                             if (val != 0 || JS7_ISDEC(c)) {
        1621             702 :                                 if (!ReportStrictModeError(cx, this, NULL, NULL,
        1622             702 :                                                            JSMSG_DEPRECATED_OCTAL)) {
        1623               0 :                                     goto error;
        1624                 :                                 }
        1625             702 :                                 setOctalCharacterEscape();
        1626                 :                             }
        1627             738 :                             if ('0' <= c && c < '8') {
        1628             702 :                                 val = 8 * val + JS7_UNDEC(c);
        1629             702 :                                 getChar();
        1630             702 :                                 c = peekChar();
        1631             702 :                                 if ('0' <= c && c < '8') {
        1632             702 :                                     int32_t save = val;
        1633             702 :                                     val = 8 * val + JS7_UNDEC(c);
        1634             702 :                                     if (val <= 0377)
        1635             702 :                                         getChar();
        1636                 :                                     else
        1637               0 :                                         val = save;
        1638                 :                                 }
        1639                 :                             }
        1640                 :
        1641             738 :                             c = (jschar)val;
        1642          133875 :                         } else if (c == 'u') {
        1643                 :                             jschar cp[4];
        1644          103032 :                             if (peekChars(4, cp) &&
        1645           45792 :                                 JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1]) &&
        1646           45792 :                                 JS7_ISHEX(cp[2]) && JS7_ISHEX(cp[3])) {
        1647           22896 :                                 c = (((((JS7_UNHEX(cp[0]) << 4)
        1648           22896 :                                         + JS7_UNHEX(cp[1])) << 4)
        1649           22896 :                                       + JS7_UNHEX(cp[2])) << 4)
        1650           68688 :                                     + JS7_UNHEX(cp[3]);
        1651           11448 :                                 skipChars(4);
        1652                 :                             } else {
        1653                 :                                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1654               0 :                                                          JSMSG_MALFORMED_ESCAPE, "Unicode");
        1655               0 :                                 goto error;
        1656                 :                             }
        1657          122427 :                         } else if (c == 'x') {
        1658                 :                             jschar cp[2];
        1659            1395 :                             if (peekChars(2, cp) &&
        1660            1116 :                                 JS7_ISHEX(cp[0]) && JS7_ISHEX(cp[1])) {
        1661             279 :                                 c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
        1662             279 :                                 skipChars(2);
        1663                 :                             } else {
        1664                 :                                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1665               0 :                                                          JSMSG_MALFORMED_ESCAPE, "hexadecimal");
        1666               0 :                                 goto error;
        1667                 :                             }
        1668          122148 :                         } else if (c == '\n') {
        1669                 :                             /*
        1670                 :                              * ES5 7.8.4: an escaped line terminator represents
        1671                 :                              * no character.
        1672                 :                              */
        1673           20025 :                             continue;
        1674                 :                         }
        1675          114588 :                         break;
        1676                 :                     }
        1677          338929 :                 } else if (TokenBuf::isRawEOLChar(c) || c == EOF) {
        1678               0 :                     ungetCharIgnoreEOL(c);
        1679                 :                     ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1680               0 :                                              JSMSG_UNTERMINATED_STRING);
        1681               0 :                     goto error;
        1682                 :                 }
        1683                 :             }
        1684         9719479 :             if (!tokenbuf.append(c))
        1685               0 :                 goto error;
        1686                 :         }
        1687          297429 :         JSAtom *atom = atomize(cx, tokenbuf);
        1688          297429 :         if (!atom)
        1689               0 :             goto error;
        1690          297429 :         tp->pos.end.lineno = lineno;
        1691          297429 :         tp->setAtom(JSOP_STRING, atom);
        1692          297429 :         tt = TOK_STRING;
        1693          297429 :         goto out;
        1694                 :     }
        1695                 :
        1696                 :     /*
        1697                 :      * Look for a decimal number.
        1698                 :      */
        1699         2262719 :     if (c1kind == Dec) {
        1700          311967 :         numStart = userbuf.addressOfNextRawChar() - 1;
        1701                 :
        1702                 :       decimal:
        1703          394602 :         hasFracOrExp = false;
        1704         1198344 :         while (JS7_ISDEC(c))
        1705          409140 :             c = getCharIgnoreEOL();
        1706                 :
        1707          394602 :         if (c == '.') {
        1708                 :           decimal_dot:
        1709            5995 :             hasFracOrExp = true;
        1710           26975 :             do {
        1711           26975 :                 c = getCharIgnoreEOL();
        1712                 :             } while (JS7_ISDEC(c));
        1713                 :         }
        1714          394782 :         if (c == 'e' || c == 'E') {
        1715            1134 :             hasFracOrExp = true;
        1716            1134 :             c = getCharIgnoreEOL();
        1717            1134 :             if (c == '+' || c == '-')
        1718             387 :                 c = getCharIgnoreEOL();
        1719            1134 :             if (!JS7_ISDEC(c)) {
        1720               0 :                 ungetCharIgnoreEOL(c);
        1721                 :                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        1722               0 :                                          JSMSG_MISSING_EXPONENT);
        1723               0 :                 goto error;
        1724                 :             }
        1725            2196 :             do {
        1726            2196 :                 c = getCharIgnoreEOL();
        1727                 :             } while (JS7_ISDEC(c));
        1728                 :         }
        1729          394782 :         ungetCharIgnoreEOL(c);
        1730                 :
        1731          394782 :         if (c != EOF && IsIdentifierStart(c)) {
        1732               0 :             ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_IDSTART_AFTER_NUMBER);
        1733               0 :             goto error;
        1734                 :         }
        1735                 :
        1736                 :         /*
        1737                 :          * Unlike identifiers and strings, numbers cannot contain escaped
        1738                 :          * chars, so we don't need to use tokenbuf.  Instead we can just
        1739                 :          * convert the jschars in userbuf directly to the numeric value.
        1740                 :          */
        1741                 :         double dval;
        1742                 :         const jschar *dummy;
        1743          394782 :         if (!hasFracOrExp) {
        1744          388121 :             if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), 10, &dummy, &dval))
        1745               0 :                 goto error;
        1746                 :         } else {
        1747            6661 :             if (!js_strtod(cx, numStart, userbuf.addressOfNextRawChar(), &dummy, &dval))
        1748               0 :                 goto error;
        1749                 :         }
        1750          394782 :         tp->setNumber(dval);
        1751          394782 :         tt = TOK_NUMBER;
        1752          394782 :         goto out;
        1753                 :     }
        1754                 :
        1755         1950752 :     if (c1kind == Colon) {
        1756                 : #if JS_HAS_XML_SUPPORT
        1757           35402 :         if (matchChar(':')) {
        1758              72 :             tt = TOK_DBLCOLON;
        1759              72 :             goto out;
        1760                 :         }
        1761                 : #endif
        1762           35330 :         tp->t_op = JSOP_NOP;
        1763           35330 :         tt = TOK_COLON;
        1764           35330 :         goto out;
        1765                 :     }
        1766                 :
        1767         1915350 :     if (c1kind == Plus) {
        1768         1584426 :         if (matchChar('=')) {
        1769           27437 :             tp->t_op = JSOP_ADD;
        1770           27437 :             tt = TOK_ADDASSIGN;
        1771         1556989 :         } else if (matchChar('+')) {
        1772           28762 :             tt = TOK_INC;
        1773                 :         } else {
        1774         1528227 :             tp->t_op = JSOP_POS;
        1775         1528227 :             tt = TOK_PLUS;
        1776                 :         }
        1777         1584426 :         goto out;
        1778                 :     }
        1779                 :
        1780                 :     /*
        1781                 :      * Look for a hexadecimal or octal number.
        1782                 :      */
        1783          330924 :     if (c1kind == HexOct) {
        1784                 :         int radix;
        1785           87972 :         c = getCharIgnoreEOL();
        1786           87972 :         if (c == 'x' || c == 'X') {
        1787            5328 :             radix = 16;
        1788            5328 :             c = getCharIgnoreEOL();
        1789            5328 :             if (!JS7_ISHEX(c)) {
        1790               0 :                 ungetCharIgnoreEOL(c);
        1791               0 :                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_MISSING_HEXDIGITS);
        1792               0 :                 goto error;
        1793                 :             }
        1794            5328 :             numStart = userbuf.addressOfNextRawChar() - 1;  /* one past the '0x' */
        1795           32049 :             while (JS7_ISHEX(c))
        1796           21393 :                 c = getCharIgnoreEOL();
        1797           82644 :         } else if (JS7_ISDEC(c)) {
        1798               9 :             radix = 8;
        1799               9 :             numStart = userbuf.addressOfNextRawChar() - 1;  /* one past the '0' */
        1800              45 :             while (JS7_ISDEC(c)) {
        1801                 :                 /* Octal integer literals are not permitted in strict mode code. */
        1802              27 :                 if (!ReportStrictModeError(cx, this, NULL, NULL, JSMSG_DEPRECATED_OCTAL))
        1803               0 :                     goto error;
        1804                 :
        1805                 :                 /*
        1806                 :                  * Outside strict mode, we permit 08 and 09 as decimal numbers,
        1807                 :                  * which makes our behaviour a superset of the ECMA numeric
        1808                 :                  * grammar. We might not always be so permissive, so we warn
        1809                 :                  * about it.
        1810                 :                  */
        1811              27 :                 if (c >= '8') {
        1812               0 :                     if (!ReportCompileErrorNumber(cx, this, NULL, JSREPORT_WARNING,
        1813               0 :                                                   JSMSG_BAD_OCTAL, c == '8' ? "08" : "09")) {
        1814               0 :                         goto error;
        1815                 :                     }
        1816               0 :                     goto decimal;   /* use the decimal scanner for the rest of the number */
        1817                 :                 }
        1818              27 :                 c = getCharIgnoreEOL();
        1819                 :             }
        1820                 :         } else {
        1821                 :             /* '0' not followed by 'x', 'X' or a digit;  scan as a decimal number. */
        1822           82635 :             numStart = userbuf.addressOfNextRawChar() - 1;
        1823           82635 :             goto decimal;
        1824                 :         }
        1825            5337 :         ungetCharIgnoreEOL(c);
        1826                 :
        1827            5337 :         if (c != EOF && IsIdentifierStart(c)) {
        1828               0 :             ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_IDSTART_AFTER_NUMBER);
        1829               0 :             goto error;
        1830                 :         }
        1831                 :
        1832                 :         double dval;
        1833                 :         const jschar *dummy;
        1834            5337 :         if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, &dummy, &dval))
        1835               0 :             goto error;
        1836            5337 :         tp->setNumber(dval);
        1837            5337 :         tt = TOK_NUMBER;
        1838            5337 :         goto out;
        1839                 :     }
        1840                 :
        1841                 :     /*
        1842                 :      * This handles everything else.
        1843                 :      */
        1844          242952 :     JS_ASSERT(c1kind == Other);
        1845          242952 :     switch (c) {
        1846                 :       case '\\':
        1847               0 :         hadUnicodeEscape = matchUnicodeEscapeIdStart(&qc);
        1848               0 :         if (hadUnicodeEscape) {
        1849               0 :             identStart = userbuf.addressOfNextRawChar() - 6;
        1850               0 :             goto identifier;
        1851                 :         }
        1852               0 :         goto badchar;
        1853                 :
        1854                 :       case '|':
        1855            4662 :         if (matchChar(c)) {
        1856            2907 :             tt = TOK_OR;
        1857            1755 :         } else if (matchChar('=')) {
        1858             126 :             tp->t_op = JSOP_BITOR;
        1859             126 :             tt = TOK_BITORASSIGN;
        1860                 :         } else {
        1861            1629 :             tt = TOK_BITOR;
        1862                 :         }
        1863            4662 :         break;
        1864                 :
        1865                 :       case '^':
        1866            1683 :         if (matchChar('=')) {
        1867              54 :             tp->t_op = JSOP_BITXOR;
        1868              54 :             tt = TOK_BITXORASSIGN;
        1869                 :         } else {
        1870            1629 :             tt = TOK_BITXOR;
        1871                 :         }
        1872            1683 :         break;
        1873                 :
        1874                 :       case '&':
        1875            6777 :         if (matchChar('&')) {
        1876            3870 :             tt = TOK_AND;
        1877            2907 :         } else if (matchChar('=')) {
        1878              36 :             tp->t_op = JSOP_BITAND;
        1879              36 :             tt = TOK_BITANDASSIGN;
        1880                 :         } else {
        1881            2871 :             tt = TOK_BITAND;
        1882                 :         }
        1883            6777 :         break;
        1884                 :
        1885                 :       case '!':
        1886           84786 :         if (matchChar('=')) {
        1887            8167 :             if (matchChar('=')) {
        1888            4834 :                 tp->t_op = JSOP_STRICTNE;
        1889            4834 :                 tt = TOK_STRICTNE;
        1890                 :             } else {
        1891            3333 :                 tp->t_op = JSOP_NE;
        1892            3333 :                 tt = TOK_NE;
        1893                 :             }
        1894                 :         } else {
        1895           76619 :             tp->t_op = JSOP_NOT;
        1896           76619 :             tt = TOK_NOT;
        1897                 :         }
        1898           84786 :         break;
        1899                 :
        1900                 : #if JS_HAS_XML_SUPPORT
        1901                 :       case '@':
        1902              18 :         tt = TOK_AT;
        1903              18 :         break;
        1904                 : #endif
        1905                 :
        1906                 :       case '<':
        1907                 : #if JS_HAS_XML_SUPPORT
        1908           34541 :         if ((flags & TSF_OPERAND) && !isStrictMode() && (hasXML() || peekChar() != '!')) {
        1909            4362 :             if (!getXMLMarkup(&tt, &tp))
        1910               0 :                 goto error;
        1911            4362 :             goto out;
        1912                 :         }
        1913                 : #endif
        1914                 :
        1915                 :         /* NB: treat HTML begin-comment as comment-till-end-of-line */
        1916           30179 :         if (matchChar('!')) {
        1917               9 :             if (matchChar('-')) {
        1918               0 :                 if (matchChar('-')) {
        1919               0 :                     flags |= TSF_IN_HTML_COMMENT;
        1920               0 :                     goto skipline;
        1921                 :                 }
        1922               0 :                 ungetChar('-');
        1923                 :             }
        1924               9 :             ungetChar('!');
        1925                 :         }
        1926           30179 :         if (matchChar('<')) {
        1927            2061 :             tp->t_op = JSOP_LSH;
        1928            2061 :             tt = matchChar('=') ? TOK_LSHASSIGN : TOK_LSH;
        1929                 :         } else {
        1930           28118 :             if (matchChar('=')) {
        1931            1683 :                 tp->t_op = JSOP_LE;
        1932            1683 :                 tt = TOK_LE;
        1933                 :             } else {
        1934           26435 :                 tp->t_op = JSOP_LT;
        1935           26435 :                 tt = TOK_LT;
        1936                 :             }
        1937                 :         }
        1938           30179 :         break;
        1939                 :
        1940                 :       case '>':
        1941            9091 :         if (matchChar('>')) {
        1942            3754 :             if (matchChar('>')) {
        1943            1386 :                 tp->t_op = JSOP_URSH;
        1944            1386 :                 tt = matchChar('=') ? TOK_URSHASSIGN : TOK_URSH;
        1945                 :             } else {
        1946            2368 :                 tp->t_op = JSOP_RSH;
        1947            2368 :                 tt = matchChar('=') ? TOK_RSHASSIGN : TOK_RSH;
        1948                 :             }
        1949                 :         } else {
        1950            5337 :             if (matchChar('=')) {
        1951            2250 :                 tp->t_op = JSOP_GE;
        1952            2250 :                 tt = TOK_GE;
        1953                 :             } else {
        1954            3087 :                 tp->t_op = JSOP_GT;
        1955            3087 :                 tt = TOK_GT;
        1956                 :             }
        1957                 :         }
        1958            9091 :         break;
        1959                 :
        1960                 :       case '*':
        1961            4923 :         tp->t_op = JSOP_MUL;
        1962            4923 :         tt = matchChar('=') ? TOK_MULASSIGN : TOK_STAR;
        1963            4923 :         break;
        1964                 :
        1965                 :       case '/':
        1966                 :         /*
        1967                 :          * Look for a single-line comment.
        1968                 :          */
        1969           58789 :         if (matchChar('/')) {
        1970           24808 :             if (cx->hasAtLineOption() && !getAtLine())
        1971               0 :                 goto error;
        1972                 :
        1973           24808 :             if (!getAtSourceMappingURL())
        1974               0 :                 goto error;
        1975                 :
        1976                 :   skipline:
        1977                 :             /* Optimize line skipping if we are not in an HTML comment. */
        1978           24808 :             if (flags & TSF_IN_HTML_COMMENT) {
        1979               0 :                 while ((c = getChar()) != EOF && c != '\n') {
        1980               0 :                     if (c == '-' && matchChar('-') && matchChar('>'))
        1981               0 :                         flags &= ~TSF_IN_HTML_COMMENT;
        1982                 :                 }
        1983                 :             } else {
        1984         1071294 :                 while ((c = getChar()) != EOF && c != '\n')
        1985         1021678 :                     continue;
        1986                 :             }
        1987           24808 :             ungetChar(c);
        1988           24808 :             cursor = (cursor - 1) & ntokensMask;
        1989           24808 :             goto retry;
        1990                 :         }
        1991                 :
        1992                 :         /*
        1993                 :          * Look for a multi-line comment.
        1994                 :          */
        1995           33981 :         if (matchChar('*')) {
        1996           26514 :             unsigned linenoBefore = lineno;
        1997         1999629 :             while ((c = getChar()) != EOF &&
        1998           79587 :                    !(c == '*' && matchChar('/'))) {
        1999                 :                 /* Ignore all characters until comment close. */
        2000                 :             }
        2001           26514 :             if (c == EOF) {
        2002                 :                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        2003               0 :                                          JSMSG_UNTERMINATED_COMMENT);
        2004               0 :                 goto error;
        2005                 :             }
        2006           26514 :             if (linenoBefore != lineno)
        2007            3366 :                 updateFlagsForEOL();
        2008           26514 :             cursor = (cursor - 1) & ntokensMask;
        2009           26514 :             goto retry;
        2010                 :         }
        2011                 :
        2012                 :         /*
        2013                 :          * Look for a regexp.
        2014                 :          */
        2015            7467 :         if (flags & TSF_OPERAND) {
        2016            3686 :             tokenbuf.clear();
        2017                 :
        2018            3686 :             bool inCharClass = false;
        2019           42082 :             for (;;) {
        2020           45768 :                 c = getChar();
        2021           45768 :                 if (c == '\\') {
        2022            4482 :                     if (!tokenbuf.append(c))
        2023               0 :                         goto error;
        2024            4482 :                     c = getChar();
        2025           41286 :                 } else if (c == '[') {
        2026            1215 :                     inCharClass = true;
        2027           40071 :                 } else if (c == ']') {
        2028            1206 :                     inCharClass = false;
        2029           38865 :                 } else if (c == '/' && !inCharClass) {
        2030                 :                     /* For compat with IE, allow unescaped / in char classes. */
        2031                 :                     break;
        2032                 :                 }
        2033           42082 :                 if (c == '\n' || c == EOF) {
        2034               0 :                     ungetChar(c);
        2035                 :                     ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR,
        2036               0 :                                              JSMSG_UNTERMINATED_REGEXP);
        2037               0 :                     goto error;
        2038                 :                 }
        2039           42082 :                 if (!tokenbuf.append(c))
        2040               0 :                     goto error;
        2041                 :             }
        2042                 :
        2043            3686 :             RegExpFlag reflags = NoFlags;
        2044            3686 :             unsigned length = tokenbuf.length() + 1;
        2045            2172 :             while (true) {
        2046            5858 :                 c = peekChar();
        2047            5858 :                 if (c == 'g' && !(reflags & GlobalFlag))
        2048            1451 :                     reflags = RegExpFlag(reflags | GlobalFlag);
        2049            4407 :                 else if (c == 'i' && !(reflags & IgnoreCaseFlag))
        2050             694 :                     reflags = RegExpFlag(reflags | IgnoreCaseFlag);
        2051            3713 :                 else if (c == 'm' && !(reflags & MultilineFlag))
        2052               0 :                     reflags = RegExpFlag(reflags | MultilineFlag);
        2053            3713 :                 else if (c == 'y' && !(reflags & StickyFlag))
        2054              27 :                     reflags = RegExpFlag(reflags | StickyFlag);
        2055                 :                 else
        2056                 :                     break;
        2057            2172 :                 getChar();
        2058            2172 :                 length++;
        2059                 :             }
        2060                 :
        2061            3686 :             c = peekChar();
        2062            7372 :             if (JS7_ISLET(c)) {
        2063               0 :                 char buf[2] = { '\0', '\0' };
        2064               0 :                 tp->pos.begin.index += length + 1;
        2065               0 :                 buf[0] = char(c);
        2066                 :                 ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_BAD_REGEXP_FLAG,
        2067               0 :                                          buf);
        2068               0 :                 (void) getChar();
        2069               0 :                 goto error;
        2070                 :             }
        2071            3686 :             tp->setRegExpFlags(reflags);
        2072            3686 :             tt = TOK_REGEXP;
        2073            3686 :             break;
        2074                 :         }
        2075                 :
        2076            3781 :         tp->t_op = JSOP_DIV;
        2077            3781 :         tt = matchChar('=') ? TOK_DIVASSIGN : TOK_DIV;
        2078            3781 :         break;
        2079                 :
        2080                 :       case '%':
        2081            1125 :         tp->t_op = JSOP_MOD;
        2082            1125 :         tt = matchChar('=') ? TOK_MODASSIGN : TOK_MOD;
        2083            1125 :         break;
        2084                 :
        2085                 :       case '~':
        2086             135 :         tp->t_op = JSOP_BITNOT;
        2087             135 :         tt = TOK_BITNOT;
        2088             135 :         break;
        2089                 :
        2090                 :       case '-':
        2091           36404 :         if (matchChar('=')) {
        2092             315 :             tp->t_op = JSOP_SUB;
        2093             315 :             tt = TOK_SUBASSIGN;
        2094           36089 :         } else if (matchChar(c)) {
        2095            1369 :             if (peekChar() == '>' && !(flags & TSF_DIRTYLINE)) {
        2096               0 :                 flags &= ~TSF_IN_HTML_COMMENT;
        2097               0 :                 goto skipline;
        2098                 :             }
        2099            1369 :             tt = TOK_DEC;
        2100                 :         } else {
        2101           34720 :             tp->t_op = JSOP_NEG;
        2102           34720 :             tt = TOK_MINUS;
        2103                 :         }
        2104           36404 :         break;
        2105                 :
        2106                 :       badchar:
        2107                 :       default:
        2108              18 :         ReportCompileErrorNumber(cx, this, NULL, JSREPORT_ERROR, JSMSG_ILLEGAL_CHARACTER);
        2109              18 :         goto error;
        2110                 :     }
        2111                 :
        2112                 :   out:
        2113        11589210 :     flags |= TSF_DIRTYLINE;
        2114        11589210 :     tp->pos.end.index = userbuf.addressOfNextRawChar() - linebase;
        2115        11589210 :     tp->type = tt;
        2116        11589210 :     JS_ASSERT(IsTokenSane(tp));
        2117        11589210 :     return tt;
        2118                 :
        2119                 :   error:
        2120                 :     /*
        2121                 :      * For erroneous multi-line tokens we won't have changed end.lineno (it'll
        2122                 :      * still be equal to begin.lineno) so we revert end.index to be equal to
        2123                 :      * begin.index + 1 (as if it's a 1-char token) to avoid having inconsistent
        2124                 :      * begin/end positions.  end.index isn't used in error messages anyway.
        2125                 :      */
        2126              18 :     flags |= TSF_DIRTYLINE;
        2127              18 :     tp->pos.end.index = tp->pos.begin.index + 1;
        2128              18 :     tp->type = TOK_ERROR;
        2129              18 :     JS_ASSERT(IsTokenSane(tp));
        2130                 : #ifdef DEBUG
        2131                 :     /*
        2132                 :      * Poisoning userbuf on error establishes an invariant: once an erroneous
        2133                 :      * token has been seen, userbuf will not be consulted again.  This is true
        2134                 :      * because the parser will either (a) deal with the TOK_ERROR token by
        2135                 :      * aborting parsing immediately; or (b) if the TOK_ERROR token doesn't
        2136                 :      * match what it expected, it will unget the token, and the next getToken()
        2137                 :      * call will immediately return the just-gotten TOK_ERROR token again
        2138                 :      * without consulting userbuf, thanks to the lookahead buffer.
        2139                 :      */
        2140              18 :     userbuf.poison();
        2141                 : #endif
        2142              18 :     return TOK_ERROR;
        2143                 : }
        2144                 :
        2145                 : JS_FRIEND_API(int)
        2146               0 : js_fgets(char *buf, int size, FILE *file)
        2147                 : {
        2148                 :     int n, i, c;
        2149                 :     JSBool crflag;
        2150                 :
        2151               0 :     n = size - 1;
        2152               0 :     if (n < 0)
        2153               0 :         return -1;
        2154                 :
        2155               0 :     crflag = JS_FALSE;
        2156               0 :     for (i = 0; i < n && (c = fast_getc(file)) != EOF; i++) {
        2157               0 :         buf[i] = c;
        2158               0 :         if (c == '\n') {        /* any \n ends a line */
        2159               0 :             i++;                /* keep the \n; we know there is room for \0 */
        2160               0 :             break;
        2161                 :         }
        2162               0 :         if (crflag) {           /* \r not followed by \n ends line at the \r */
        2163               0 :             ungetc(c, file);
        2164               0 :             break;              /* and overwrite c in buf with \0 */
        2165                 :         }
        2166               0 :         crflag = (c == '\r');
        2167                 :     }
        2168                 :
        2169               0 :     buf[i] = '\0';
        2170               0 :     return i;
        2171                 : }
        2172                 :
        2173                 : #ifdef DEBUG
        2174                 : const char *
        2175               0 : TokenKindToString(TokenKind tt)
        2176                 : {
        2177               0 :     switch (tt) {
        2178               0 :       case TOK_ERROR:           return "TOK_ERROR";
        2179               0 :       case TOK_EOF:             return "TOK_EOF";
        2180               0 :       case TOK_EOL:             return "TOK_EOL";
        2181               0 :       case TOK_SEMI:            return "TOK_SEMI";
        2182               0 :       case TOK_COMMA:           return "TOK_COMMA";
        2183               0 :       case TOK_HOOK:            return "TOK_HOOK";
        2184               0 :       case TOK_COLON:           return "TOK_COLON";
        2185               0 :       case TOK_OR:              return "TOK_OR";
        2186               0 :       case TOK_AND:             return "TOK_AND";
        2187               0 :       case TOK_BITOR:           return "TOK_BITOR";
        2188               0 :       case TOK_BITXOR:          return "TOK_BITXOR";
        2189               0 :       case TOK_BITAND:          return "TOK_BITAND";
        2190               0 :       case TOK_PLUS:            return "TOK_PLUS";
        2191               0 :       case TOK_MINUS:           return "TOK_MINUS";
        2192               0 :       case TOK_STAR:            return "TOK_STAR";
        2193               0 :       case TOK_DIV:             return "TOK_DIV";
        2194               0 :       case TOK_MOD:             return "TOK_MOD";
        2195               0 :       case TOK_INC:             return "TOK_INC";
        2196               0 :       case TOK_DEC:             return "TOK_DEC";
        2197               0 :       case TOK_DOT:             return "TOK_DOT";
        2198               0 :       case TOK_LB:              return "TOK_LB";
        2199               0 :       case TOK_RB:              return "TOK_RB";
        2200               0 :       case TOK_LC:              return "TOK_LC";
        2201               0 :       case TOK_RC:              return "TOK_RC";
        2202               0 :       case TOK_LP:              return "TOK_LP";
        2203               0 :       case TOK_RP:              return "TOK_RP";
        2204               0 :       case TOK_NAME:            return "TOK_NAME";
        2205               0 :       case TOK_NUMBER:          return "TOK_NUMBER";
        2206               0 :       case TOK_STRING:          return "TOK_STRING";
        2207               0 :       case TOK_REGEXP:          return "TOK_REGEXP";
        2208               0 :       case TOK_TRUE:            return "TOK_TRUE";
        2209               0 :       case TOK_FALSE:           return "TOK_FALSE";
        2210               0 :       case TOK_NULL:            return "TOK_NULL";
        2211               0 :       case TOK_THIS:            return "TOK_THIS";
        2212               0 :       case TOK_FUNCTION:        return "TOK_FUNCTION";
        2213               0 :       case TOK_IF:              return "TOK_IF";
        2214               0 :       case TOK_ELSE:            return "TOK_ELSE";
        2215               0 :       case TOK_SWITCH:          return "TOK_SWITCH";
        2216               0 :       case TOK_CASE:            return "TOK_CASE";
        2217               0 :       case TOK_DEFAULT:         return "TOK_DEFAULT";
        2218               0 :       case TOK_WHILE:           return "TOK_WHILE";
        2219               0 :       case TOK_DO:              return "TOK_DO";
        2220               0 :       case TOK_FOR:             return "TOK_FOR";
        2221               0 :       case TOK_BREAK:           return "TOK_BREAK";
        2222               0 :       case TOK_CONTINUE:        return "TOK_CONTINUE";
        2223               0 :       case TOK_IN:              return "TOK_IN";
        2224               0 :       case TOK_VAR:             return "TOK_VAR";
        2225               0 :       case TOK_CONST:           return "TOK_CONST";
        2226               0 :       case TOK_WITH:            return "TOK_WITH";
        2227               0 :       case TOK_RETURN:          return "TOK_RETURN";
        2228               0 :       case TOK_NEW:             return "TOK_NEW";
        2229               0 :       case TOK_DELETE:          return "TOK_DELETE";
        2230               0 :       case TOK_TRY:             return "TOK_TRY";
        2231               0 :       case TOK_CATCH:           return "TOK_CATCH";
        2232               0 :       case TOK_FINALLY:         return "TOK_FINALLY";
        2233               0 :       case TOK_THROW:           return "TOK_THROW";
        2234               0 :       case TOK_INSTANCEOF:      return "TOK_INSTANCEOF";
        2235               0 :       case TOK_DEBUGGER:        return "TOK_DEBUGGER";
        2236               0 :       case TOK_XMLSTAGO:        return "TOK_XMLSTAGO";
        2237               0 :       case TOK_XMLETAGO:        return "TOK_XMLETAGO";
        2238               0 :       case TOK_XMLPTAGC:        return "TOK_XMLPTAGC";
        2239               0 :       case TOK_XMLTAGC:         return "TOK_XMLTAGC";
        2240               0 :       case TOK_XMLNAME:         return "TOK_XMLNAME";
        2241               0 :       case TOK_XMLATTR:         return "TOK_XMLATTR";
        2242               0 :       case TOK_XMLSPACE:        return "TOK_XMLSPACE";
        2243               0 :       case TOK_XMLTEXT:         return "TOK_XMLTEXT";
        2244               0 :       case TOK_XMLCOMMENT:      return "TOK_XMLCOMMENT";
        2245               0 :       case TOK_XMLCDATA:        return "TOK_XMLCDATA";
        2246               0 :       case TOK_XMLPI:           return "TOK_XMLPI";
        2247               0 :       case TOK_AT:              return "TOK_AT";
        2248               0 :       case TOK_DBLCOLON:        return "TOK_DBLCOLON";
        2249               0 :       case TOK_DBLDOT:          return "TOK_DBLDOT";
        2250               0 :       case TOK_FILTER:          return "TOK_FILTER";
        2251               0 :       case TOK_XMLELEM:         return "TOK_XMLELEM";
        2252               0 :       case TOK_XMLLIST:         return "TOK_XMLLIST";
        2253               0 :       case TOK_YIELD:           return "TOK_YIELD";
        2254               0 :       case TOK_LEXICALSCOPE:    return "TOK_LEXICALSCOPE";
        2255               0 :       case TOK_LET:             return "TOK_LET";
        2256               0 :       case TOK_RESERVED:        return "TOK_RESERVED";
        2257               0 :       case TOK_STRICT_RESERVED: return "TOK_STRICT_RESERVED";
        2258               0 :       case TOK_STRICTEQ:        return "TOK_STRICTEQ";
        2259               0 :       case TOK_EQ:              return "TOK_EQ";
        2260               0 :       case TOK_STRICTNE:        return "TOK_STRICTNE";
        2261               0 :       case TOK_NE:              return "TOK_NE";
        2262               0 :       case TOK_TYPEOF:          return "TOK_TYPEOF";
        2263               0 :       case TOK_VOID:            return "TOK_VOID";
        2264               0 :       case TOK_NOT:             return "TOK_NOT";
        2265               0 :       case TOK_BITNOT:          return "TOK_BITNOT";
        2266               0 :       case TOK_LT:              return "TOK_LT";
        2267               0 :       case TOK_LE:              return "TOK_LE";
        2268               0 :       case TOK_GT:              return "TOK_GT";
        2269               0 :       case TOK_GE:              return "TOK_GE";
        2270               0 :       case TOK_LSH:             return "TOK_LSH";
        2271               0 :       case TOK_RSH:             return "TOK_RSH";
        2272               0 :       case TOK_URSH:            return "TOK_URSH";
        2273               0 :       case TOK_ASSIGN:          return "TOK_ASSIGN";
        2274               0 :       case TOK_ADDASSIGN:       return "TOK_ADDASSIGN";
        2275               0 :       case TOK_SUBASSIGN:       return "TOK_SUBASSIGN";
        2276               0 :       case TOK_BITORASSIGN:     return "TOK_BITORASSIGN";
        2277               0 :       case TOK_BITXORASSIGN:    return "TOK_BITXORASSIGN";
        2278               0 :       case TOK_BITANDASSIGN:    return "TOK_BITANDASSIGN";
        2279               0 :       case TOK_LSHASSIGN:       return "TOK_LSHASSIGN";
        2280               0 :       case TOK_RSHASSIGN:       return "TOK_RSHASSIGN";
        2281               0 :       case TOK_URSHASSIGN:      return "TOK_URSHASSIGN";
        2282               0 :       case TOK_MULASSIGN:       return "TOK_MULASSIGN";
        2283               0 :       case TOK_DIVASSIGN:       return "TOK_DIVASSIGN";
        2284               0 :       case TOK_MODASSIGN:       return "TOK_MODASSIGN";
        2285               0 :       case TOK_LIMIT:           break;
        2286                 :     }
        2287                 :
        2288               0 :     return "<bad TokenKind>";
        2289                 : }
        2290                 : #endif

Generated by: LCOV version 1.7