View Javadoc

    1   /**
    2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
    3    */
    4   package net.sourceforge.pmd.cpd;
    5
    6   import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
    7   import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
    8   import net.sourceforge.pmd.cpd.cppast.Token;
    9   import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
    10
    11  import java.io.StringReader;
    12
    13  public class CPPTokenizer implements Tokenizer {
    14      private static SimpleCharStream charStream;
    15
    16      public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
    17          StringBuffer sb = sourceCode.getCodeBuffer();
    18          try {
    19              if (charStream == null) {
    20                  charStream = new SimpleCharStream(new StringReader(sb.toString()));
    21              } else {
    22                  charStream.ReInit(new StringReader(sb.toString()));
    23              }
    24              CPPParserTokenManager.ReInit(charStream);
    25              CPPParserTokenManager.setFileName(sourceCode.getFileName());
    26              Token currToken = CPPParserTokenManager.getNextToken();
    27              while (currToken.image.length() > 0) {
    28                  tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
    29                  currToken = CPPParserTokenManager.getNextToken();
    30              }
    31              tokenEntries.add(TokenEntry.getEOF());
    32              System.err.println("Added " + sourceCode.getFileName());
    33          } catch (TokenMgrError err) {
    34              err.printStackTrace();
    35              System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
    36              tokenEntries.add(TokenEntry.getEOF());
    37          }
    38      }
    39  }