(src []byte)
| 65 | } |
| 66 | |
| 67 | func tokenize(src []byte) []tokinfo { |
| 68 | fs := token.NewFileSet() |
| 69 | var s scanner.Scanner |
| 70 | s.Init(fs.AddFile("src", fs.Base(), len(src)), src, nil, scanner.ScanComments) |
| 71 | var info []tokinfo |
| 72 | for { |
| 73 | pos, next, lit := s.Scan() |
| 74 | switch next { |
| 75 | case token.SEMICOLON: |
| 76 | continue |
| 77 | } |
| 78 | info = append(info, tokinfo{ |
| 79 | pos: int(pos - 1), |
| 80 | end: int(pos + token.Pos(len(lit)) - 1), |
| 81 | Token: next, |
| 82 | }) |
| 83 | if next == token.EOF { |
| 84 | break |
| 85 | } |
| 86 | } |
| 87 | return info |
| 88 | } |