include/comma/parser/Lexer.h

Go to the documentation of this file.

//===-- parser/Lexer.h ---------------------------------------- -*- C++ -*-===//
//
// This file is distributed under the MIT license.  See LICENSE.txt for details.
//
// Copyright (C) 2008, Stephen Wilson
//
//===----------------------------------------------------------------------===//

#ifndef COMMA_PARSER_LEXER_HDR_GUARD
#define COMMA_PARSER_LEXER_HDR_GUARD

#include "comma/basic/Diagnostic.h"
#include "comma/basic/TextProvider.h"
#include <iosfwd>
#include <string>

namespace comma {

class Lexer {

public:
    Lexer(TextProvider &txtProvider, Diagnostic &diag);

    // Aside from UNUSED_ID, the only codes which should be used by clients of
    // this class are those prefixed by TKN.  All other codes are considered
    // internal and are subject to change.  Questions about a codes kind (for
    // example, determining if a code represents a reserved word) should be
    // answered thru the supplied predicates and not be based on this
    // enumerations ordering.
    enum Code {
        UNUSED_ID,

#define KEYWORD(NAME, STRING) TKN_ ## NAME,
#define GLYPH(NAME, STRING)   TKN_ ## NAME,
#define TOKEN(NAME)           TKN_ ## NAME,
#include "comma/parser/Tokens.def"
#undef KEYWORD
#undef GLYPH
#undef TOKEN

        NUMTOKEN_CODES
    };

    // The Token class represents the result of lexing process.  Tokens are
    // identified by code.  They provide access to their underlying string
    // representation, and have position information in the form of a single
    // Location entry (which must be interpreted with respect to a particular
    // TextProvider).
    class Token {

    public:
        Token() : code(Lexer::UNUSED_ID) { }

        Lexer::Code getCode() const { return code; }

        Location getLocation() const { return location; }

        const char *getRep() const { return string; }

        unsigned getLength() const { return length; }

        // This method provides a string representation of the token. If the
        // token does not have a string representation, NULL is returned.
        const char *getString() const;

    private:
        Lexer::Code code   : 8;
        unsigned    length : 24;
        Location    location;
        const char *string;

        // Declare Lexer as a friend to give access to the following
        // constructor.
        friend class Lexer;

        Token(Lexer::Code code,
              Location    location,
              const char *string,
              unsigned length)
            : code(code),
              length(length),
              location(location),
              string(string) { }
    };

    // Scans a single token from the input stream.  When the stream is
    // exhausted, all further calls to this method will set the supplied tokens
    // code to TKN_EOT.
    void scan(Lexer::Token &tkn);

    // Returns true if the lexer has seen an error while scanning its input.
    bool seenError() const { return errorDetected; }

    // Returns true if the given token is a glyph and it can name a function
    // (e.g. '+', '*', etc).
    static bool isFunctionGlyph(const Lexer::Token &tkn) {
        switch (tkn.getCode()) {
        case TKN_EQUAL:
        case TKN_MINUS:
        case TKN_NEQUAL:
        case TKN_STAR:
        case TKN_PLUS:
        case TKN_TILDE:
            return true;
        default:
            return false;
        }
    }

    // Returns a static string representation of the given token code, or NULL
    // if no such representation is available.
    static const char *tokenString(Code code);

    // Returns the string representation of the given token, or NULL if no such
    // representation is available.
    static const char *tokenString(const Token &tkn);

private:
    bool eatWhitespace();

    bool eatComment();

    bool scanWord();

    bool scanGlyph();

    bool scanString();

    bool scanNumeric();

    bool scanEscape();

    static bool isAlphabetic(unsigned c);

    static bool isInitialIdentifierChar(unsigned c);

    static bool isInnerIdentifierChar(unsigned c);

    static bool isWhitespace(unsigned c);

    static bool isDecimalDigit(unsigned c);

    Location currentLocation() const;

    // We do not read from the supplied stream directly.  Rather, we call the
    // following simple wrappers which ensure that our line and column
    // information is consistently updated.  Carriage returns and DOS style
    // newlines are canonicalized to single newline characters.  When the stream
    // is exhausted, 0 is returned to signal EOF.
    unsigned readStream();
    unsigned peekStream();
    void ungetStream();
    void ignoreStream();

    // Returns the token code assoicated with the chatacter string delimited by
    // start and end.  If the string exactly matches a reserved word (this
    // function will not recignize glyph tokens) the words corresponding code is
    // returned, else UNUSED_ID if there is no match.
    Code getTokenCode(TextIterator &start, TextIterator &end) const;

    void emitToken(Code code,
                   const TextIterator &start, const TextIterator &end);

    // Used to emit keywords and glyphs which do not require a string
    // representation.
    void emitToken(Code code, Location loc);

    void emitStringToken(const TextIterator &start, const TextIterator &end);

    void emitIntegerToken(const TextIterator &start, const TextIterator &end);

    void emitIdentifierToken(const TextIterator &start,
                             const TextIterator &end);

    DiagnosticStream &report(Location loc, diag::Kind kind) {
        SourceLocation sloc = txtProvider.getSourceLocation(loc);
        return diagnostic.report(sloc, kind);
    }

    DiagnosticStream &report(SourceLocation sloc, diag::Kind kind)  {
        return diagnostic.report(sloc, kind);
    }

    DiagnosticStream &report(diag::Kind kind) {
        SourceLocation sloc = txtProvider.getSourceLocation(currentLocation());
        return diagnostic.report(sloc, kind);
    }

    // This is the stream we read from.
    TextProvider &txtProvider;

    // Output stream to send messages.
    Diagnostic &diagnostic;

    // An iterator into our stream.
    TextIterator currentIter;

    // Flag indicating if a lexical error has been detected.
    bool errorDetected;

    // The token parameter supplied to scan() is maintained here.  This is
    // the destination of the lexing methods.
    Token *targetToken;
};

} // End comma namespace

#endif

---