1 package net.sourceforge.pmd.cpd;
2
3 import net.sourceforge.pmd.PMD;
4 import net.sourceforge.pmd.ast.JavaCharStream;
5 import net.sourceforge.pmd.ast.JavaParserTokenManager;
6 import net.sourceforge.pmd.ast.Token;
7
8 import java.io.IOException;
9 import java.io.LineNumberReader;
10 import java.io.Reader;
11 import java.io.StringReader;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 public class JavaTokenizer implements Tokenizer {
16
17 public void tokenize(SourceCode tokens, Tokens tokenEntries, Reader input) throws IOException {
18 // first get a snapshot of the code
19 List lines = new ArrayList();
20 StringBuffer sb = new StringBuffer();
21 LineNumberReader r = new LineNumberReader(input);
22 String currentLine;
23 while ((currentLine = r.readLine()) != null) {
24 lines.add(currentLine);
25 sb.append(currentLine);
26 sb.append(PMD.EOL);
27 }
28 tokens.setCode(lines);
29
30 // now tokenize it
31 /*
32 I'm doing a sort of State pattern thing here where
33 this goes into "discarding" mode when it hits an import or package
34 keyword and goes back into "accumulate mode when it hits a semicolon.
35 This could probably be turned into some objects.
36 */
37 JavaCharStream javaStream = new JavaCharStream(new StringReader(sb.toString()));
38 JavaParserTokenManager tokenMgr = new JavaParserTokenManager(javaStream);
39 Token currToken = tokenMgr.getNextToken();
40 boolean discarding = false;
41 int count = 0;
42 while (currToken.image != "") {
43 if (currToken.image.equals("import") || currToken.image.equals("package")) {
44 discarding = true;
45 currToken = tokenMgr.getNextToken();
46 continue;
47 }
48
49 if (discarding && currToken.image.equals(";")) {
50 discarding = false;
51 }
52
53 if (discarding) {
54 currToken = tokenMgr.getNextToken();
55 continue;
56 }
57
58 if (!currToken.image.equals(";")) {
59 count++;
60 tokenEntries.add(new TokenEntry(currToken.image, count, tokens.getFileName(), currToken.beginLine));
61 }
62
63 currToken = tokenMgr.getNextToken();
64 }
65 tokenEntries.add(TokenEntry.EOF);
66 }
67 }
This page was automatically generated by Maven