1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.StringReader;
7
8 import net.sourceforge.pmd.lang.LanguageVersion;
9 import net.sourceforge.pmd.lang.LanguageVersionHandler;
10 import net.sourceforge.pmd.lang.TokenManager;
11 import net.sourceforge.pmd.lang.ast.TokenMgrError;
12 import net.sourceforge.pmd.lang.cpp.ast.Token;
13 import net.sourceforge.pmd.util.IOUtil;
14
15 public class CPPTokenizer implements Tokenizer {
16
17 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
18 StringBuilder buffer = sourceCode.getCodeBuffer();
19 StringReader reader = null;
20 try {
21 LanguageVersionHandler languageVersionHandler = LanguageVersion.CPP.getLanguageVersionHandler();
22 reader = new StringReader(buffer.toString());
23 TokenManager tokenManager = languageVersionHandler.getParser(
24 languageVersionHandler.getDefaultParserOptions())
25 .getTokenManager(sourceCode.getFileName(), reader);
26 Token currentToken = (Token) tokenManager.getNextToken();
27 while (currentToken.image.length() > 0) {
28 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
29 currentToken = (Token) tokenManager.getNextToken();
30 }
31 tokenEntries.add(TokenEntry.getEOF());
32 System.err.println("Added " + sourceCode.getFileName());
33 } catch (TokenMgrError err) {
34 err.printStackTrace();
35 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
36 tokenEntries.add(TokenEntry.getEOF());
37 } finally {
38 IOUtil.closeQuietly(reader);
39 }
40 }
41 }