// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package scanner provides a scanner and tokenizer for UTF-8-encoded text. // It takes an io.Reader providing the source, which then can be tokenized // through repeated calls to the Scan function. For compatibility with // existing tools, the NUL character is not allowed. If the first character // in the source is a UTF-8 encoded byte order mark (BOM), it is discarded. // // By default, a Scanner skips white space and Go comments and recognizes all // literals as defined by the Go language specification. It may be // customized to recognize only a subset of those literals and to recognize // different identifier and white space characters.
package scanner import ( "bytes" "fmt" "io" "os" "unicode" "unicode/utf8" ) // A source position is represented by a Position value. // A position is valid if Line > 0. type Position struct { Filename string // filename, if any Offset int // byte offset, starting at 0 Line int // line number, starting at 1 Column int // column number, starting at 1 (character count per line) } // IsValid reports whether the position is valid. func ( *Position) () bool { return .Line > 0 } func ( Position) () string { := .Filename if == "" { = "<input>" } if .IsValid() { += fmt.Sprintf(":%d:%d", .Line, .Column) } return } // Predefined mode bits to control recognition of tokens. For instance, // to configure a Scanner such that it only recognizes (Go) identifiers, // integers, and skips comments, set the Scanner's Mode field to: // // ScanIdents | ScanInts | SkipComments // // With the exceptions of comments, which are skipped if SkipComments is // set, unrecognized tokens are not ignored. Instead, the scanner simply // returns the respective individual characters (or possibly sub-tokens). // For instance, if the mode is ScanIdents (not ScanStrings), the string // "foo" is scanned as the token sequence '"' Ident '"'. // // Use GoTokens to configure the Scanner such that it accepts all Go // literal tokens including Go identifiers. Comments will be skipped. // const ( ScanIdents = 1 << -Ident ScanInts = 1 << -Int ScanFloats = 1 << -Float // includes Ints and hexadecimal floats ScanChars = 1 << -Char ScanStrings = 1 << -String ScanRawStrings = 1 << -RawString ScanComments = 1 << -Comment SkipComments = 1 << -skipComment // if set with ScanComments, comments become white space GoTokens = ScanIdents | ScanFloats | ScanChars | ScanStrings | ScanRawStrings | ScanComments | SkipComments ) // The result of Scan is one of these tokens or a Unicode character. const ( EOF = -(iota + 1) Ident Int Float Char String RawString Comment // internal use only skipComment ) var tokenString = map[rune]string{ EOF: "EOF", Ident: "Ident", Int: "Int", Float: "Float", Char: "Char", String: "String", RawString: "RawString", Comment: "Comment", } // TokenString returns a printable string for a token or Unicode character. func ( rune) string { if , := tokenString[]; { return } return fmt.Sprintf("%q", string()) } // GoWhitespace is the default value for the Scanner's Whitespace field. // Its value selects Go's white space characters. const GoWhitespace = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' ' const bufLen = 1024 // at least utf8.UTFMax // A Scanner implements reading of Unicode characters and tokens from an io.Reader. type Scanner struct { // Input src io.Reader // Source buffer srcBuf [bufLen + 1]byte // +1 for sentinel for common case of s.next() srcPos int // reading position (srcBuf index) srcEnd int // source end (srcBuf index) // Source position srcBufOffset int // byte offset of srcBuf[0] in source line int // line count column int // character count lastLineLen int // length of last line in characters (for correct column reporting) lastCharLen int // length of last character in bytes // Token text buffer // Typically, token text is stored completely in srcBuf, but in general // the token text's head may be buffered in tokBuf while the token text's // tail is stored in srcBuf. tokBuf bytes.Buffer // token text head that is not in srcBuf anymore tokPos int // token text tail position (srcBuf index); valid if >= 0 tokEnd int // token text tail end (srcBuf index) // One character look-ahead ch rune // character before current srcPos // Error is called for each error encountered. If no Error // function is set, the error is reported to os.Stderr. Error func(s *Scanner, msg string) // ErrorCount is incremented by one for each error encountered. ErrorCount int // The Mode field controls which tokens are recognized. For instance, // to recognize Ints, set the ScanInts bit in Mode. The field may be // changed at any time. Mode uint // The Whitespace field controls which characters are recognized // as white space. To recognize a character ch <= ' ' as white space, // set the ch'th bit in Whitespace (the Scanner's behavior is undefined // for values ch > ' '). The field may be changed at any time. Whitespace uint64 // IsIdentRune is a predicate controlling the characters accepted // as the ith rune in an identifier. The set of valid characters // must not intersect with the set of white space characters. // If no IsIdentRune function is set, regular Go identifiers are // accepted instead. The field may be changed at any time. IsIdentRune func(ch rune, i int) bool // Start position of most recently scanned token; set by Scan. // Calling Init or Next invalidates the position (Line == 0). // The Filename field is always left untouched by the Scanner. // If an error is reported (via Error) and Position is invalid, // the scanner is not inside a token. Call Pos to obtain an error // position in that case, or to obtain the position immediately // after the most recently scanned token. Position } // Init initializes a Scanner with a new source and returns s. // Error is set to nil, ErrorCount is set to 0, Mode is set to GoTokens, // and Whitespace is set to GoWhitespace. func ( *Scanner) ( io.Reader) *Scanner { .src = // initialize source buffer // (the first call to next() will fill it by calling src.Read) .srcBuf[0] = utf8.RuneSelf // sentinel .srcPos = 0 .srcEnd = 0 // initialize source position .srcBufOffset = 0 .line = 1 .column = 0 .lastLineLen = 0 .lastCharLen = 0 // initialize token text buffer // (required for first call to next()). .tokPos = -1 // initialize one character look-ahead .ch = -2 // no char read yet, not EOF // initialize public fields .Error = nil .ErrorCount = 0 .Mode = GoTokens .Whitespace = GoWhitespace .Line = 0 // invalidate token position return } // next reads and returns the next Unicode character. It is designed such // that only a minimal amount of work needs to be done in the common ASCII // case (one test to check for both ASCII and end-of-buffer, and one test // to check for newlines). func ( *Scanner) () rune { , := rune(.srcBuf[.srcPos]), 1 if >= utf8.RuneSelf { // uncommon case: not ASCII or not enough bytes for .srcPos+utf8.UTFMax > .srcEnd && !utf8.FullRune(.srcBuf[.srcPos:.srcEnd]) { // not enough bytes: read some more, but first // save away token text if any if .tokPos >= 0 { .tokBuf.Write(.srcBuf[.tokPos:.srcPos]) .tokPos = 0 // s.tokEnd is set by Scan() } // move unread bytes to beginning of buffer copy(.srcBuf[0:], .srcBuf[.srcPos:.srcEnd]) .srcBufOffset += .srcPos // read more bytes // (an io.Reader must return io.EOF when it reaches // the end of what it is reading - simply returning // n == 0 will make this loop retry forever; but the // error is in the reader implementation in that case) := .srcEnd - .srcPos , := .src.Read(.srcBuf[:bufLen]) .srcPos = 0 .srcEnd = + .srcBuf[.srcEnd] = utf8.RuneSelf // sentinel if != nil { if != io.EOF { .error(.Error()) } if .srcEnd == 0 { if .lastCharLen > 0 { // previous character was not EOF .column++ } .lastCharLen = 0 return EOF } // If err == EOF, we won't be getting more // bytes; break to avoid infinite loop. If // err is something else, we don't know if // we can get more bytes; thus also break. break } } // at least one byte = rune(.srcBuf[.srcPos]) if >= utf8.RuneSelf { // uncommon case: not ASCII , = utf8.DecodeRune(.srcBuf[.srcPos:.srcEnd]) if == utf8.RuneError && == 1 { // advance for correct error position .srcPos += .lastCharLen = .column++ .error("invalid UTF-8 encoding") return } } } // advance .srcPos += .lastCharLen = .column++ // special situations switch { case 0: // for compatibility with other tools .error("invalid character NUL") case '\n': .line++ .lastLineLen = .column .column = 0 } return } // Next reads and returns the next Unicode character. // It returns EOF at the end of the source. It reports // a read error by calling s.Error, if not nil; otherwise // it prints an error message to os.Stderr. Next does not // update the Scanner's Position field; use Pos() to // get the current position. func ( *Scanner) () rune { .tokPos = -1 // don't collect token text .Line = 0 // invalidate token position := .Peek() if != EOF { .ch = .next() } return } // Peek returns the next Unicode character in the source without advancing // the scanner. It returns EOF if the scanner's position is at the last // character of the source. func ( *Scanner) () rune { if .ch == -2 { // this code is only run for the very first character .ch = .next() if .ch == '\uFEFF' { .ch = .next() // ignore BOM } } return .ch } func ( *Scanner) ( string) { .tokEnd = .srcPos - .lastCharLen // make sure token text is terminated .ErrorCount++ if .Error != nil { .Error(, ) return } := .Position if !.IsValid() { = .Pos() } fmt.Fprintf(os.Stderr, "%s: %s\n", , ) } func ( *Scanner) ( string, ...interface{}) { .error(fmt.Sprintf(, ...)) } func ( *Scanner) ( rune, int) bool { if .IsIdentRune != nil { return .IsIdentRune(, ) } return == '_' || unicode.IsLetter() || unicode.IsDigit() && > 0 } func ( *Scanner) () rune { // we know the zero'th rune is OK; start scanning at the next one := .next() for := 1; .isIdentRune(, ); ++ { = .next() } return } func lower( rune) rune { return ('a' - 'A') | } // returns lower-case ch iff ch is ASCII letter func isDecimal( rune) bool { return '0' <= && <= '9' } func isHex( rune) bool { return '0' <= && <= '9' || 'a' <= lower() && lower() <= 'f' } // digits accepts the sequence { digit | '_' } starting with ch0. // If base <= 10, digits accepts any decimal digit but records // the first invalid digit >= base in *invalid if *invalid == 0. // digits returns the first rune that is not part of the sequence // anymore, and a bitset describing whether the sequence contained // digits (bit 0 is set), or separators '_' (bit 1 is set). func ( *Scanner) ( rune, int, *rune) ( rune, int) { = if <= 10 { := rune('0' + ) for isDecimal() || == '_' { := 1 if == '_' { = 2 } else if >= && * == 0 { * = } |= = .next() } } else { for isHex() || == '_' { := 1 if == '_' { = 2 } |= = .next() } } return } func ( *Scanner) ( rune, bool) (rune, rune) { := 10 // number base := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b' := 0 // bit 0: digit present, bit 1: '_' present := rune(0) // invalid digit in literal, or 0 // integer part var rune var int if ! { = Int if == '0' { = .next() switch lower() { case 'x': = .next() , = 16, 'x' case 'o': = .next() , = 8, 'o' case 'b': = .next() , = 2, 'b' default: , = 8, '0' = 1 // leading 0 } } , = .digits(, , &) |= if == '.' && .Mode&ScanFloats != 0 { = .next() = true } } // fractional part if { = Float if == 'o' || == 'b' { .error("invalid radix point in " + litname()) } , = .digits(, , &) |= } if &1 == 0 { .error(litname() + " has no digits") } // exponent if := lower(); ( == 'e' || == 'p') && .Mode&ScanFloats != 0 { switch { case == 'e' && != 0 && != '0': .errorf("%q exponent requires decimal mantissa", ) case == 'p' && != 'x': .errorf("%q exponent requires hexadecimal mantissa", ) } = .next() = Float if == '+' || == '-' { = .next() } , = .digits(, 10, nil) |= if &1 == 0 { .error("exponent has no digits") } } else if == 'x' && == Float { .error("hexadecimal mantissa requires a 'p' exponent") } if == Int && != 0 { .errorf("invalid digit %q in %s", , litname()) } if &2 != 0 { .tokEnd = .srcPos - .lastCharLen // make sure token text is terminated if := invalidSep(.TokenText()); >= 0 { .error("'_' must separate successive digits") } } return , } func litname( rune) string { switch { default: return "decimal literal" case 'x': return "hexadecimal literal" case 'o', '0': return "octal literal" case 'b': return "binary literal" } } // invalidSep returns the index of the first invalid separator in x, or -1. func invalidSep( string) int { := ' ' // prefix char, we only care if it's 'x' := '.' // digit, one of '_', '0' (a digit), or '.' (anything else) := 0 // a prefix counts as a digit if len() >= 2 && [0] == '0' { = lower(rune([1])) if == 'x' || == 'o' || == 'b' { = '0' = 2 } } // mantissa and exponent for ; < len(); ++ { := // previous digit = rune([]) switch { case == '_': if != '0' { return } case isDecimal() || == 'x' && isHex(): = '0' default: if == '_' { return - 1 } = '.' } } if == '_' { return len() - 1 } return -1 } func digitVal( rune) int { switch { case '0' <= && <= '9': return int( - '0') case 'a' <= lower() && lower() <= 'f': return int(lower() - 'a' + 10) } return 16 // larger than any legal digit val } func ( *Scanner) ( rune, , int) rune { for > 0 && digitVal() < { = .next() -- } if > 0 { .error("invalid char escape") } return } func ( *Scanner) ( rune) rune { := .next() // read character after '/' switch { case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', : // nothing to do = .next() case '0', '1', '2', '3', '4', '5', '6', '7': = .scanDigits(, 8, 3) case 'x': = .scanDigits(.next(), 16, 2) case 'u': = .scanDigits(.next(), 16, 4) case 'U': = .scanDigits(.next(), 16, 8) default: .error("invalid char escape") } return } func ( *Scanner) ( rune) ( int) { := .next() // read character after quote for != { if == '\n' || < 0 { .error("literal not terminated") return } if == '\\' { = .scanEscape() } else { = .next() } ++ } return } func ( *Scanner) () { := .next() // read character after '`' for != '`' { if < 0 { .error("literal not terminated") return } = .next() } } func ( *Scanner) () { if .scanString('\'') != 1 { .error("invalid char literal") } } func ( *Scanner) ( rune) rune { // ch == '/' || ch == '*' if == '/' { // line comment = .next() // read character after "//" for != '\n' && >= 0 { = .next() } return } // general comment = .next() // read character after "/*" for { if < 0 { .error("comment not terminated") break } := = .next() if == '*' && == '/' { = .next() break } } return } // Scan reads the next token or Unicode character from source and returns it. // It only recognizes tokens t for which the respective Mode bit (1<<-t) is set. // It returns EOF at the end of the source. It reports scanner errors (read and // token errors) by calling s.Error, if not nil; otherwise it prints an error // message to os.Stderr. func ( *Scanner) () rune { := .Peek() // reset token text position .tokPos = -1 .Line = 0 : // skip white space for .Whitespace&(1<<uint()) != 0 { = .next() } // start collecting token text .tokBuf.Reset() .tokPos = .srcPos - .lastCharLen // set token position // (this is a slightly optimized version of the code in Pos()) .Offset = .srcBufOffset + .tokPos if .column > 0 { // common case: last character was not a '\n' .Line = .line .Column = .column } else { // last character was a '\n' // (we cannot be at the beginning of the source // since we have called next() at least once) .Line = .line - 1 .Column = .lastLineLen } // determine token value := switch { case .isIdentRune(, 0): if .Mode&ScanIdents != 0 { = Ident = .scanIdentifier() } else { = .next() } case isDecimal(): if .Mode&(ScanInts|ScanFloats) != 0 { , = .scanNumber(, false) } else { = .next() } default: switch { case EOF: break case '"': if .Mode&ScanStrings != 0 { .scanString('"') = String } = .next() case '\'': if .Mode&ScanChars != 0 { .scanChar() = Char } = .next() case '.': = .next() if isDecimal() && .Mode&ScanFloats != 0 { , = .scanNumber(, true) } case '/': = .next() if ( == '/' || == '*') && .Mode&ScanComments != 0 { if .Mode&SkipComments != 0 { .tokPos = -1 // don't collect token text = .scanComment() goto } = .scanComment() = Comment } case '`': if .Mode&ScanRawStrings != 0 { .scanRawString() = RawString } = .next() default: = .next() } } // end of token text .tokEnd = .srcPos - .lastCharLen .ch = return } // Pos returns the position of the character immediately after // the character or token returned by the last call to Next or Scan. // Use the Scanner's Position field for the start position of the most // recently scanned token. func ( *Scanner) () ( Position) { .Filename = .Filename .Offset = .srcBufOffset + .srcPos - .lastCharLen switch { case .column > 0: // common case: last character was not a '\n' .Line = .line .Column = .column case .lastLineLen > 0: // last character was a '\n' .Line = .line - 1 .Column = .lastLineLen default: // at the beginning of the source .Line = 1 .Column = 1 } return } // TokenText returns the string corresponding to the most recently scanned token. // Valid after calling Scan and in calls of Scanner.Error. func ( *Scanner) () string { if .tokPos < 0 { // no token text return "" } if .tokEnd < .tokPos { // if EOF was reached, s.tokEnd is set to -1 (s.srcPos == 0) .tokEnd = .tokPos } // s.tokEnd >= s.tokPos if .tokBuf.Len() == 0 { // common case: the entire token text is still in srcBuf return string(.srcBuf[.tokPos:.tokEnd]) } // part of the token text was saved in tokBuf: save the rest in // tokBuf as well and return its content .tokBuf.Write(.srcBuf[.tokPos:.tokEnd]) .tokPos = .tokEnd // ensure idempotency of TokenText() call return .tokBuf.String() }