Function scanTokens

hclsyntax/scan_tokens.go:4222–5273

line scan_tokens.rl:18

(data []byte, filename string, start hcl.Pos, mode scanMode)

//line scan_tokens.rl:18

func scanTokens(data []byte, filename string, start hcl.Pos, mode scanMode) []Token {
	stripData := stripUTF8BOM(data)
	start.Byte += len(data) - len(stripData)
	data = stripData

	f := &tokenAccum{
		Filename:  filename,
		Bytes:     data,
		Pos:       start,
		StartByte: start.Byte,
	}

//line scan_tokens.rl:317

	// Ragel state
	p := 0          // "Pointer" into data
	pe := len(data) // End-of-data "pointer"
	ts := 0
	te := 0
	act := 0
	eof := pe
	var stack []int
	var top int

	var cs int // current state
	switch mode {
	case scanNormal:
		cs = hcltok_en_main
	case scanTemplate:
		cs = hcltok_en_bareTemplate
	case scanIdentOnly:
		cs = hcltok_en_identOnly
	default:
		panic("invalid scanMode")
	}

	braces := 0
	var retBraces []int              // stack of brace levels that cause us to use fret
	var heredocs []heredocInProgress // stack of heredocs we're currently processing

//line scan_tokens.rl:352

	// Make Go compiler happy
	_ = ts
	_ = te
	_ = act
	_ = eof

	token := func(ty TokenType) {
		f.emitToken(ty, ts, te)
	}
	selfToken := func() {
		b := data[ts:te]
		if len(b) != 1 {
			// should never happen
			panic("selfToken only works for single-character tokens")
		}
		f.emitToken(TokenType(b[0]), ts, te)