View Javadoc

    1   /**
    2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
    3    */
    4   package net.sourceforge.pmd.cpd;
    5
    6   import java.io.StringReader;
    7
    8   import net.sourceforge.pmd.lang.LanguageVersion;
    9   import net.sourceforge.pmd.lang.LanguageVersionHandler;
    10  import net.sourceforge.pmd.lang.TokenManager;
    11  import net.sourceforge.pmd.lang.ast.TokenMgrError;
    12  import net.sourceforge.pmd.lang.cpp.ast.Token;
    13  import net.sourceforge.pmd.util.IOUtil;
    14
    15  public class CPPTokenizer implements Tokenizer {
    16
    17  	public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    18  		StringBuilder buffer = sourceCode.getCodeBuffer();
    19  		StringReader reader = null;
    20  		try {
    21  			LanguageVersionHandler languageVersionHandler = LanguageVersion.CPP.getLanguageVersionHandler();
    22  			reader = new StringReader(buffer.toString());
    23  			TokenManager tokenManager = languageVersionHandler.getParser(
    24  					languageVersionHandler.getDefaultParserOptions())
    25  					.getTokenManager(sourceCode.getFileName(), reader);
    26  			Token currentToken = (Token) tokenManager.getNextToken();
    27  			while (currentToken.image.length() > 0) {
    28  				tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
    29  				currentToken = (Token) tokenManager.getNextToken();
    30  			}
    31  			tokenEntries.add(TokenEntry.getEOF());
    32  			System.err.println("Added " + sourceCode.getFileName());
    33  		} catch (TokenMgrError err) {
    34  			err.printStackTrace();
    35  			System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
    36  			tokenEntries.add(TokenEntry.getEOF());
    37  		} finally {
    38  			IOUtil.closeQuietly(reader);
    39  		}
    40  	}
    41  }