-
Notifications
You must be signed in to change notification settings - Fork 1
/
Tokenizer.cpp
36 lines (29 loc) · 836 Bytes
/
Tokenizer.cpp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#include "Tokenizer.h"
#include "clang/Frontend/CompilerInstance.h"
#include "llvm/Support/MemoryBuffer.h"
#include "clang/Lex/Lexer.h"
namespace Tokenizer
{
TokenVector tokenizeString(const std::string &string, clang::CompilerInstance &instance)
{
// Try to lex it
llvm::MemoryBuffer *buffer = llvm::MemoryBuffer::getMemBuffer(llvm::StringRef(string));
clang::FileID fileId = instance.getSourceManager().createFileIDForMemBuffer(buffer);
clang::Lexer lex(fileId, buffer, instance.getSourceManager(), clang::LangOptions());
lex.SetKeepWhitespaceMode(false);
// Fetch all of the tokens
TokenVector tokens;
while(true)
{
clang::Token token;
lex.LexFromRawLexer(token);
if (token.getKind() == clang::tok::eof)
{
// We're done
break;
}
tokens.push_back(token);
}
return tokens;
}
}