View Javadoc

1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import java.io.StringReader;
7   
8   import net.sourceforge.pmd.lang.LanguageVersion;
9   import net.sourceforge.pmd.lang.LanguageVersionHandler;
10  import net.sourceforge.pmd.lang.TokenManager;
11  import net.sourceforge.pmd.lang.ast.TokenMgrError;
12  import net.sourceforge.pmd.lang.cpp.ast.Token;
13  import net.sourceforge.pmd.util.IOUtil;
14  
15  public class CPPTokenizer implements Tokenizer {
16  
17  	public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
18  		StringBuilder buffer = sourceCode.getCodeBuffer();
19  		StringReader reader = null;
20  		try {
21  			LanguageVersionHandler languageVersionHandler = LanguageVersion.CPP.getLanguageVersionHandler();
22  			reader = new StringReader(buffer.toString());
23  			TokenManager tokenManager = languageVersionHandler.getParser(
24  					languageVersionHandler.getDefaultParserOptions())
25  					.getTokenManager(sourceCode.getFileName(), reader);
26  			Token currentToken = (Token) tokenManager.getNextToken();
27  			while (currentToken.image.length() > 0) {
28  				tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
29  				currentToken = (Token) tokenManager.getNextToken();
30  			}
31  			tokenEntries.add(TokenEntry.getEOF());
32  			System.err.println("Added " + sourceCode.getFileName());
33  		} catch (TokenMgrError err) {
34  			err.printStackTrace();
35  			System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
36  			tokenEntries.add(TokenEntry.getEOF());
37  		} finally {
38  			IOUtil.closeQuietly(reader);
39  		}
40  	}
41  }