当前位置：源码 > Java源码 >

Java源码示例：org.apache.flink.streaming.api.operators.Output

示例1

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

示例2

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}

示例3

private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

示例4

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

示例5

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	if (selectedOutputs.isEmpty()) {
		return;
	}

	Iterator<Output<StreamRecord<OUT>>> it = selectedOutputs.iterator();

	while (true) {
		Output<StreamRecord<OUT>> out = it.next();
		if (it.hasNext()) {
			// we don't have the last output
			// perform a shallow copy
			StreamRecord<OUT> shallowCopy = record.copy(record.getValue());
			out.collect(shallowCopy);
		} else {
			// this is the last output
			out.collect(record);
			break;
		}
	}
}

示例6

private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

示例7

@Override
public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) {
	super.setup(containingTask, config, output);

	this.inStreamElementSerializer = new StreamElementSerializer<>(
		getOperatorConfig().<IN>getTypeSerializerIn1(getUserCodeClassloader()));

	switch (outputMode) {
		case ORDERED:
			queue = new OrderedStreamElementQueue<>(capacity);
			break;
		case UNORDERED:
			queue = new UnorderedStreamElementQueue<>(capacity);
			break;
		default:
			throw new IllegalStateException("Unknown async mode: " + outputMode + '.');
	}

	this.timestampedCollector = new TimestampedCollector<>(output);
}

示例8

protected Set<Output<StreamRecord<OUT>>> selectOutputs(StreamRecord<OUT> record)  {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = new HashSet<>(selectAllOutputs.length);
	Collections.addAll(selectedOutputs, selectAllOutputs);

	for (OutputSelector<OUT> outputSelector : outputSelectors) {
		Iterable<String> outputNames = outputSelector.select(record.getValue());

		for (String outputName : outputNames) {
			Output<StreamRecord<OUT>>[] outputList = outputMap.get(outputName);
			if (outputList != null) {
				Collections.addAll(selectedOutputs, outputList);
			}
		}
	}

	return selectedOutputs;
}

示例9

BufferedOutputManager(
    Output<StreamRecord<WindowedValue<OutputT>>> output,
    TupleTag<OutputT> mainTag,
    Map<TupleTag<?>, OutputTag<WindowedValue<?>>> tagsToOutputTags,
    Map<TupleTag<?>, Integer> tagsToIds,
    Lock bufferLock,
    PushedBackElementsHandler<KV<Integer, WindowedValue<?>>> pushedBackElementsHandler) {
  this.output = output;
  this.mainTag = mainTag;
  this.tagsToOutputTags = tagsToOutputTags;
  this.tagsToIds = tagsToIds;
  this.bufferLock = bufferLock;
  this.idsToTags = new HashMap<>();
  for (Map.Entry<TupleTag<?>, Integer> entry : tagsToIds.entrySet()) {
    idsToTags.put(entry.getValue(), entry.getKey());
  }
  this.pushedBackElementsHandler = pushedBackElementsHandler;
}

示例10

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}

示例11

public Emitter(
		final Object checkpointLock,
		final Output<StreamRecord<OUT>> output,
		final StreamElementQueue streamElementQueue,
		final OperatorActions operatorActions) {

	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "checkpointLock");
	this.output = Preconditions.checkNotNull(output, "output");
	this.streamElementQueue = Preconditions.checkNotNull(streamElementQueue, "streamElementQueue");
	this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions");

	this.timestampedCollector = new TimestampedCollector<>(this.output);
	this.running = true;
}

示例12

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}

示例13

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		TimerService timerService) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setTimerService(timerService)
			.build();

		operator.setProcessingTimeService(mockTask.getProcessingTimeServiceFactory().createProcessingTimeService(null));
		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

示例14

AsyncDataOutputToOutput(
		Output<StreamRecord<T>> output,
		StreamStatusMaintainer streamStatusMaintainer) {
	super(streamStatusMaintainer);

	this.output = checkNotNull(output);
}

示例15

@Override
public void collect(StreamRecord<T> record) {

	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];
		StreamRecord<T> shallowCopy = record.copy(record.getValue());
		output.collect(shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(record);
	}
}

示例16

@SafeVarargs
private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(
		OneInputStreamOperator<T, T>... operators) {

	checkNotNull(operators);
	checkArgument(operators.length > 0);

	try (MockEnvironment env = MockEnvironment.builder().build()) {

	final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env);

		final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class);
		final StreamConfig cfg = new StreamConfig(new Configuration());

		final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length];

		// initial output goes to nowhere
		@SuppressWarnings({"unchecked", "rawtypes"})
		WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(
				new Output[0], statusProvider);

		// build the reverse operators array
		for (int i = 0; i < ops.length; i++) {
			OneInputStreamOperator<T, T> op = operators[ops.length - i - 1];
			op.setup(containingTask, cfg, lastWriter);
			lastWriter = new ChainingOutput<>(op, statusProvider, null);
			ops[i] = op;
		}

		@SuppressWarnings("unchecked")
		final OP head = (OP) operators[0];

		return new OperatorChain<>(
				ops,
				new RecordWriterOutput<?>[0],
				lastWriter,
				head);
	}
}

示例17

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}

示例18

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}

示例19

@SuppressWarnings("unchecked")
@Override
public <T extends StreamOperator<OUT>> T createStreamOperator(StreamTask<?, ?> containingTask,
		StreamConfig config, Output<StreamRecord<OUT>> output) {
	return (T) generatedClass.newInstance(containingTask.getUserCodeClassLoader(),
			generatedClass.getReferences(), containingTask, config, output);
}

示例20

@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}

示例21

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	for (Output<StreamRecord<OUT>> out : allOutputs) {
		out.emitWatermark(mark);
	}
}

示例22

@Override
public void collect(StreamRecord<OUT> record) {
	Set<Output<StreamRecord<OUT>>> selectedOutputs = selectOutputs(record);

	for (Output<StreamRecord<OUT>> out : selectedOutputs) {
		out.collect(record);
	}
}

示例23

public Emitter(
		final Object checkpointLock,
		final Output<StreamRecord<OUT>> output,
		final StreamElementQueue streamElementQueue,
		final OperatorActions operatorActions) {

	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "checkpointLock");
	this.output = Preconditions.checkNotNull(output, "output");
	this.streamElementQueue = Preconditions.checkNotNull(streamElementQueue, "streamElementQueue");
	this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions");

	this.timestampedCollector = new TimestampedCollector<>(this.output);
	this.running = true;
}

示例24

@Override
public void emitWatermark(Watermark mark) {
	watermarkGauge.setCurrentWatermark(mark.getTimestamp());
	if (streamStatusProvider.getStreamStatus().isActive()) {
		for (Output<StreamRecord<T>> output : outputs) {
			output.emitWatermark(mark);
		}
	}
}

示例25

@Override
public void collect(StreamRecord<T> record) {

	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];
		StreamRecord<T> shallowCopy = record.copy(record.getValue());
		output.collect(shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(record);
	}
}

示例26

@Override
public <X> void collect(OutputTag<X> outputTag, StreamRecord<X> record) {
	for (int i = 0; i < outputs.length - 1; i++) {
		Output<StreamRecord<T>> output = outputs[i];

		StreamRecord<X> shallowCopy = record.copy(record.getValue());
		output.collect(outputTag, shallowCopy);
	}

	if (outputs.length > 0) {
		// don't copy for the last output
		outputs[outputs.length - 1].collect(outputTag, record);
	}
}

示例27

@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector,
				OperatorChain<?, ?> operatorChain) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}

示例28

DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}

示例29

DownstreamCollector(
    MessageFactoryType messageFactoryType, Output<StreamRecord<Message>> output) {
  this.factory = MessageFactory.forType(messageFactoryType);
  this.output = Objects.requireNonNull(output);
  this.multiLanguagePayloads =
      messageFactoryType == MessageFactoryType.WITH_PROTOBUF_PAYLOADS_MULTILANG;
}

示例30

public StreamOutputHandler(String outputStreamId, TypeInformation<R> typeInfo, AbstractDefinition definition, Output<StreamRecord<R>> output) {
    this.outputStreamId = outputStreamId;
    this.typeInfo = typeInfo;
    this.definition = definition;
    this.output = output;
    this.objectMapper = new ObjectMapper();
    this.objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}

Java源码示例：org.apache.flink.streaming.api.operators.Output

微信关注