当前位置：源码 > Java源码 >

Java源码示例：org.eclipse.jdt.internal.core.util.PublicScanner

示例1

public String getIdentifierType(final PublicScanner scanner) {
	final int startPos = scanner.getCurrentTokenStartPosition();
	final int endPos = scanner.getCurrentTokenEndPosition();
	final Range<Integer> tokenRange = Range
			.closedOpen(startPos, endPos);

	final String tokenName = scanner.getCurrentTokenString();

	// TODO: Find the tightest of all
	if (isInSet(tokenName, tokenRange, variables)) {
		return IDENTIFIER_PREFIX + "_VAR";
	} else if (isInSet(tokenName, tokenRange, methods)) {
		return IDENTIFIER_PREFIX + "_METHOD";
	} else if (isInSet(tokenName, tokenRange, types)) {
		return IDENTIFIER_PREFIX + "_TYPE";
	}
	return IDENTIFIER_PREFIX + "_UNK";
}

示例2

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

示例3

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

示例4

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例5

public String getIdentifierType(final PublicScanner scanner) {
	final int startPos = scanner.getCurrentTokenStartPosition();
	final int endPos = scanner.getCurrentTokenEndPosition();
	final Range<Integer> tokenRange = Range
			.closedOpen(startPos, endPos);

	final String tokenName = scanner.getCurrentTokenString();

	// TODO: Find the tightest of all
	if (isInSet(tokenName, tokenRange, variables)) {
		return IDENTIFIER_PREFIX + "_VAR";
	} else if (isInSet(tokenName, tokenRange, methods)) {
		return IDENTIFIER_PREFIX + "_METHOD";
	} else if (isInSet(tokenName, tokenRange, types)) {
		return IDENTIFIER_PREFIX + "_TYPE";
	}
	return IDENTIFIER_PREFIX + "_UNK";
}

示例6

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

示例7

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

示例8

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例9

public String getIdentifierType(final PublicScanner scanner) {
	final int startPos = scanner.getCurrentTokenStartPosition();
	final int endPos = scanner.getCurrentTokenEndPosition();
	final Range<Integer> tokenRange = Range
			.closedOpen(startPos, endPos);

	final String tokenName = scanner.getCurrentTokenString();

	// TODO: Find the tightest of all
	if (isInSet(tokenName, tokenRange, variables)) {
		return IDENTIFIER_PREFIX + "_VAR";
	} else if (isInSet(tokenName, tokenRange, methods)) {
		return IDENTIFIER_PREFIX + "_METHOD";
	} else if (isInSet(tokenName, tokenRange, types)) {
		return IDENTIFIER_PREFIX + "_TYPE";
	}
	return IDENTIFIER_PREFIX + "_UNK";
}

示例10

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

示例11

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

示例12

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例13

private final String getTokenType(final int tokenType,
		final PublicScanner scanner, final IdentifierTypeRetriever retriever) {
	if (tokenType == ITerminalSymbols.TokenNameIdentifier) {
		return retriever.getIdentifierType(scanner);
	} else if (JavaTokenTypeTokenizer.isLiteralToken(tokenType)) {
		return LITERAL;
	} else {
		return scanner.getCurrentTokenString();
	}
}

示例14

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例15

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

示例16

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

示例17

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例18

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}

示例19

public List<WhitespaceAnnotatedToken> getTokensWithWidthData(
		final char[] code) {
	final List<WhitespaceAnnotatedToken> tokens = Lists.newArrayList();
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_START,
			SENTENCE_START, 0, 0));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				final int currentPosition = scanner
						.getCurrentTokenStartPosition();
				final int currentLine = scanner
						.getLineNumber(currentPosition);
				final int lineStart = scanner.getLineStart(currentLine);
				tokens.add(new WhitespaceAnnotatedToken(cToken, "",
						currentPosition - lineStart, scanner
						.getCurrentTokenString().length()));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_END, SENTENCE_END,
			0, 0));
	return tokens;
}

示例20

/**
 * @param code
 * @return
 */
private PublicScanner prepareScanner(final char[] code) {
	final PublicScanner scanner = new PublicScanner();
	scanner.setSource(code);
	scanner.tokenizeWhiteSpace = true;
	scanner.recordLineSeparator = true;
	scanner.tokenizeComments = true;
	whitespaceConverter = new WhitespaceToTokenConverter();
	return scanner;
}

示例21

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = prepareScanner(code);

	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner
						.getCurrentTokenStartPosition();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				int i = 0;
				final List<String> cTokens = getConvertedToken(scanner,
						token);
				for (final String cToken : cTokens) {
					tokens.put(position + i, cToken);
					i++;
				}
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());
	}
	return tokens;
}

示例22

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例23

private final String getTokenType(final int tokenType,
		final PublicScanner scanner, final IdentifierTypeRetriever retriever) {
	if (tokenType == ITerminalSymbols.TokenNameIdentifier) {
		return retriever.getIdentifierType(scanner);
	} else if (JavaTokenTypeTokenizer.isLiteralToken(tokenType)) {
		return LITERAL;
	} else {
		return scanner.getCurrentTokenString();
	}
}

示例24

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例25

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(new FullToken(IDENTIFIER_TOKEN, ""));
			} else if (isLiteralToken(token)) {
				tokens.add(new FullToken(LITERAL_TOKEN, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(new FullToken(COMMENT_BLOCK, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(new FullToken(COMMENT_JAVADOC, ""));
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(new FullToken(COMMENT_LINE, ""));
			} else {
				tokens.add(new FullToken(scanner.getCurrentTokenString(),
						""));
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}

示例26

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			} else if (token == ITerminalSymbols.TokenNameIdentifier) {
				tokens.add(IDENTIFIER_TOKEN);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
				tokens.add(COMMENT_BLOCK);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
				tokens.add(COMMENT_LINE);
			} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
				tokens.add(COMMENT_JAVADOC);
			} else if (isLiteralToken(token)) {
				tokens.add(LITERAL_TOKEN);
			} else {
				tokens.add(scanner.getCurrentTokenString());
			}

		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}

示例27

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position, IDENTIFIER_TOKEN);
				} else if (isLiteralToken(token)) {
					tokens.put(position, LITERAL_TOKEN);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, COMMENT_BLOCK);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, COMMENT_JAVADOC);
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, COMMENT_LINE);
				} else {
					tokens.put(position, scanner.getCurrentTokenString());
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}

示例28

/**
 * @param tokens
 * @param scanner
 * @param token
 * @return
 * @throws InvalidInputException
 */
private List<String> getConvertedToken(final PublicScanner scanner,
		final int token) throws InvalidInputException {
	final List<String> tokens = Lists.newArrayList();
	if (token == ITerminalSymbols.TokenNameEOF) {
		return Collections.emptyList();
	}
	final String tokenString = scanner.getCurrentTokenString();

	if (token == ITerminalSymbols.TokenNameWHITESPACE) {
		tokens.add(whitespaceConverter.toWhiteSpaceSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameIdentifier) {
		tokens.add(toIdentifierSymbol(tokenString));
	} else if (JavaTokenTypeTokenizer.isLiteralToken(token)) {
		tokens.add(toLiteralSymbol(tokenString));
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_BLOCK);
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_LINE);
		final int nextToken = scanner.getNextToken();
		if (nextToken == ITerminalSymbols.TokenNameWHITESPACE) {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"
					+ scanner.getCurrentTokenString()));
		} else {
			tokens.add(whitespaceConverter.toWhiteSpaceSymbol("\n"));
			tokens.addAll(getConvertedToken(scanner, nextToken));
		}
	} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
		tokens.add(JavaTokenTypeTokenizer.COMMENT_JAVADOC);
	} else {
		tokens.add(tokenString);
	}

	return tokens;
}

示例29

public List<WhitespaceAnnotatedToken> getTokensWithWidthData(
		final char[] code) {
	final List<WhitespaceAnnotatedToken> tokens = Lists.newArrayList();
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_START,
			SENTENCE_START, 0, 0));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				final int currentPosition = scanner
						.getCurrentTokenStartPosition();
				final int currentLine = scanner
						.getLineNumber(currentPosition);
				final int lineStart = scanner.getLineStart(currentLine);
				tokens.add(new WhitespaceAnnotatedToken(cToken, "",
						currentPosition - lineStart, scanner
						.getCurrentTokenString().length()));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_END, SENTENCE_END,
			0, 0));
	return tokens;
}

示例30

/**
 * @param code
 * @return
 */
private PublicScanner prepareScanner(final char[] code) {
	final PublicScanner scanner = new PublicScanner();
	scanner.setSource(code);
	scanner.tokenizeWhiteSpace = true;
	scanner.recordLineSeparator = true;
	scanner.tokenizeComments = true;
	whitespaceConverter = new WhitespaceToTokenConverter();
	return scanner;
}

Java源码示例：org.eclipse.jdt.internal.core.util.PublicScanner

微信关注