当前位置：源码 > Java源码 >

Java源码示例：org.python.pydev.parser.jython.FastCharStream

示例1

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(),
					new FullToken(nextToken.image, Integer
							.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例2

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<FullToken> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(new FullToken(nextToken.image, Integer
					.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例3

@Override
public List<String> tokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<String> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例4

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(), nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例5

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(),
					new FullToken(nextToken.image, Integer
							.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例6

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<FullToken> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(new FullToken(nextToken.image, Integer
					.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例7

@Override
public List<String> tokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<String> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例8

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(), nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例9

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(),
					new FullToken(nextToken.image, Integer
							.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例10

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<FullToken> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(new FullToken(nextToken.image, Integer
					.toString(nextToken.kind)));
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例11

@Override
public List<String> tokenListFromCode(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final List<String> tokens = Lists.newArrayList();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			tokens.add(nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例12

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final FastCharStream stream = new FastCharStream(code);
	final ITokenManager mng = getPythonTokenizer(stream);
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();

	Token nextToken = mng.getNextToken();
	while (nextToken.kind != 0) {
		if (shouldAdd(nextToken)) {
			// TODO: Bad Heurisitc...
			tokens.put(
					nextToken.getBeginLine() * 500
							+ nextToken.getBeginCol(), nextToken.image);
		}
		nextToken = mng.getNextToken();
	}

	return tokens;
}

示例13

public static FStringsGrammar createGrammar(String s) {

        char[] c = s.toCharArray();
        for (int i = 0, z = 0; i < c.length; i++) {
            if (c[i] == '\\' && i + 2 < c.length && c[i + 1] == 'N' && c[i + 2] == '{') {
                c[i + 2] = '_';
                i += 2;
                z = i + 1;
                for (; z < c.length; z++) {
                    if (c[z] == '}') {
                        c[z] = '_';
                        break;
                    }
                }
            }
        }

        FastCharStream in = new FastCharStream(c);
        FStringsGrammar fStringsGrammar = new FStringsGrammar(in);

        return fStringsGrammar;
    }

示例14

@Override
public FullToken getTokenFromString(final String token) {
	final FastCharStream stream = new FastCharStream(token.toCharArray());
	final ITokenManager mng = getPythonTokenizer(stream);
	final Token pyToken = mng.getNextToken();
	return new FullToken(pyToken.image, Integer.toString(pyToken.kind));
}

示例15

@Override
public FullToken getTokenFromString(final String token) {
	final FastCharStream stream = new FastCharStream(token.toCharArray());
	final ITokenManager mng = getPythonTokenizer(stream);
	final Token pyToken = mng.getNextToken();
	return new FullToken(pyToken.image, Integer.toString(pyToken.kind));
}

示例16

@Override
public FullToken getTokenFromString(final String token) {
	final FastCharStream stream = new FastCharStream(token.toCharArray());
	final ITokenManager mng = getPythonTokenizer(stream);
	final Token pyToken = mng.getNextToken();
	return new FullToken(pyToken.image, Integer.toString(pyToken.kind));
}

示例17

/** Constructor with user supplied FastCharStream. */
public FStringsGrammar(FastCharStream stream) {
  token_source = new FStringsGrammarTokenManager(stream);
  token = new Token();
  jj_ntk = -1;
  jj_gen = 0;
  for (int i = 0; i < 29; i++) jj_la1[i] = -1;
}

示例18

/**
 * Actually creates the grammar.
 * @param generateTree whether we should generate the AST or not.
 */
public static IGrammar createGrammar(boolean generateTree, int grammarVersion, char[] charArray) {
    IGrammar grammar;
    FastCharStream in = new FastCharStream(charArray);
    switch (grammarVersion) {
        case IPythonNature.GRAMMAR_PYTHON_VERSION_2_5:
            grammar = new PythonGrammar25(generateTree, in);
            break;
        case IPythonNature.GRAMMAR_PYTHON_VERSION_2_6:
            grammar = new PythonGrammar26(generateTree, in);
            break;
        case IPythonNature.GRAMMAR_PYTHON_VERSION_2_7:
            grammar = new PythonGrammar27(generateTree, in);
            break;
        case IPythonNature.GRAMMAR_PYTHON_VERSION_3_5:
            grammar = new PythonGrammar30(generateTree, in);
            break;
        case IPythonNature.GRAMMAR_PYTHON_VERSION_3_6:
        case IPythonNature.GRAMMAR_PYTHON_VERSION_3_7:
            grammar = new PythonGrammar36(generateTree, in);
            break;
        case IPythonNature.GRAMMAR_PYTHON_VERSION_3_8:
            grammar = new PythonGrammar38(generateTree, in);
            break;
        //case CYTHON: not treated here (only in reparseDocument).
        default:
            throw new RuntimeException("The grammar specified for parsing is not valid: " + grammarVersion);
    }

    if (ENABLE_TRACING) {
        //grammar has to be generated with debugging info for this to make a difference
        grammar.enable_tracing();
    }
    return grammar;
}

示例19

/**
 * Searches for a new line in the input stream. If found, it'll stop right after it, otherwise, the stream will be
 * backed up the number of chars that've been read.
 */
protected static boolean searchNewLine(ITokenManager tokenManager, boolean breakOnFirstNotWhitespace) {
    boolean foundNewLine = false;
    FastCharStream inputStream = tokenManager.getInputStream();
    int currentPos = inputStream.getCurrentPos();

    try {
        while (true) {
            try {
                char c = inputStream.readChar();
                if (c == '\r' || c == '\n') {
                    if (c == '\r') {
                        c = inputStream.readChar();
                        if (c != '\n') {
                            inputStream.backup(1);
                        }
                    }
                    foundNewLine = true;
                    break;
                }
                if (breakOnFirstNotWhitespace && !Character.isWhitespace(c)) {
                    break;
                }
            } catch (IOException e) {
                break;
            }
        }
    } finally {
        if (!foundNewLine) {
            inputStream.restorePos(currentPos);
        }
    }
    return foundNewLine;
}

示例20

@Override
public final FastCharStream getInputStream() {
    if (this.inputStream == null) {
        this.inputStream = (FastCharStream) Reflection.getAttrObj(this, "input_stream", true);
    }
    return inputStream;
}

示例21

public void testEvalInput() throws ParseException {
    String s = "" +
            "call(20, 20), 20\n" +
            "";
    FastCharStream in = new FastCharStream(s.toCharArray());
    PythonGrammar36 grammar = new PythonGrammar36(true, in);
    Expr eval_input = grammar.eval_input();
    assertEquals(
            "Expr[value=Tuple[elts=[Call[func=Name[id=call, ctx=Load, reserved=false], args=[Num[n=20, type=Int, num=20], Num[n=20, type=Int, num=20]], keywords=[], starargs=null, kwargs=null], Num[n=20, type=Int, num=20]], ctx=Load, endsWithComma=false]]",
            eval_input.toString());

}

示例22

@Override
public ITokenManager getPythonTokenizer(FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar27TokenManager(stream);
	return mng;
}

示例23

/**
 * @param stream
 * @return
 */
@Override
public ITokenManager getPythonTokenizer(final FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar30TokenManager(stream);
	return mng;
}

示例24

@Override
public ITokenManager getPythonTokenizer(FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar27TokenManager(stream);
	return mng;
}

示例25

/**
 * @param stream
 * @return
 */
@Override
public ITokenManager getPythonTokenizer(final FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar30TokenManager(stream);
	return mng;
}

示例26

@Override
public ITokenManager getPythonTokenizer(FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar27TokenManager(stream);
	return mng;
}

示例27

/**
 * @param stream
 * @return
 */
@Override
public ITokenManager getPythonTokenizer(final FastCharStream stream) {
	final ITokenManager mng = new PythonGrammar30TokenManager(stream);
	return mng;
}

示例28

/** Constructor. */
public PythonGrammar36TokenManager(FastCharStream stream){
   input_stream = stream;
}

示例29

/** Constructor. */
public PythonGrammar36TokenManager(FastCharStream stream, int lexState){
   this(stream);
   SwitchTo(lexState);
}

示例30

/** Constructor. */
public PythonGrammar26TokenManager(FastCharStream stream) {
    input_stream = stream;
}

Java源码示例：org.python.pydev.parser.jython.FastCharStream

微信关注