源码类别
热门源码
 当前位置：源码 > Java源码 >

Java源码示例：org.apache.flink.api.common.ExecutionMode

示例1
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
示例2
/**
 * This function connects the operators that produce the broadcast inputs to this operator.
 *
 * @param operatorToNode The map from program operators to optimizer nodes.
 * @param defaultExchangeMode The data exchange mode to use, if the operator does not
 *                            specify one.
 *
 * @throws CompilerException
 */
public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) {
	// skip for Operators that don't support broadcast variables 
	if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) {
		return;
	}

	// get all broadcast inputs
	AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator());

	// create connections and add them
	for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) {
		OptimizerNode predecessor = operatorToNode.get(input.getValue());
		DagConnection connection = new DagConnection(predecessor, this,
														ShipStrategyType.BROADCAST, defaultExchangeMode);
		addBroadcastConnection(input.getKey(), connection);
		predecessor.addOutgoingConnection(connection);
	}
}
示例3
@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
示例4
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
示例5
@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
示例6
private static JobGraph createJobGraph(
		ScheduleMode scheduleMode,
		ResultPartitionType resultPartitionType,
		ExecutionMode executionMode) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	JobVertex source = new JobVertex("source");
	source.setInvokableClass(LongValueSource.class);
	source.setParallelism(PARALLELISM);
	source.setSlotSharingGroup(slotSharingGroup);

	JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(ResultVerifyingSink.class);
	sink.setParallelism(PARALLELISM);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
	JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(scheduleMode);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setExecutionMode(executionMode);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
示例7
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
示例8
@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
示例9
@Test
public void testPipelineBreakerBroadcastedAllReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> bcInput1 = sourceWithMapper
									.map(new IdentityMapper<Long>())
									.reduce(new SelectOneReducer<Long>());
		DataSet<Long> bcInput2 = env.generateSequence(1, 10);
		
		DataSet<Long> result = sourceWithMapper
				.map(new IdentityMapper<Long>())
						.withBroadcastSet(bcInput1, "bc1")
						.withBroadcastSet(bcInput2, "bc2");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
示例10
public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
示例11
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
	Operator<?> children = getOperator().getInput();

	final OptimizerNode pred;
	final DagConnection conn;
	
	pred = contractToNode.get(children);
	conn = new DagConnection(pred, this, defaultExchangeMode);
		
	// create the connection and add it
	this.input = conn;
	pred.addOutgoingConnection(conn);
}
示例12
public SinkJoiner(OptimizerNode input1, OptimizerNode input2) {
	super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo()));

	DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED);
	DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED);
	
	this.input1 = conn1;
	this.input2 = conn2;
	
	setParallelism(1);
}
示例13
/**
 * Gets the data exchange mode to use for this connection.
 *
 * @return The data exchange mode to use for this connection.
 */
public ExecutionMode getDataExchangeMode() {
	if (dataExchangeMode == null) {
		throw new IllegalStateException("This connection does not have the data exchange mode set");
	}
	return dataExchangeMode;
}
示例14
@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
示例15
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchigPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
示例16
@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
示例17
@Test
public void testPipelineBreakerBroadcastedAllReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> bcInput1 = sourceWithMapper
									.map(new IdentityMapper<Long>())
									.reduce(new SelectOneReducer<Long>());
		DataSet<Long> bcInput2 = env.generateSequence(1, 10);
		
		DataSet<Long> result = sourceWithMapper
				.map(new IdentityMapper<Long>())
						.withBroadcastSet(bcInput1, "bc1")
						.withBroadcastSet(bcInput2, "bc2");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
示例18
public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
示例19
@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
示例20
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchigPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
示例21
@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchigPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH);
}
示例22
@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
示例23
/**
 * Gets the data exchange mode to use for this connection.
 *
 * @return The data exchange mode to use for this connection.
 */
public ExecutionMode getDataExchangeMode() {
	if (dataExchangeMode == null) {
		throw new IllegalStateException("This connection does not have the data exchange mode set");
	}
	return dataExchangeMode;
}
示例24
@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
}
示例25
@Test
public void testPipelineBreakerWithBroadcastVariable() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> result = source.map(new IdentityMapper<Long>())
									.map(new IdentityMapper<Long>())
										.withBroadcastSet(source, "bc");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode mapperInput = (SingleInputPlanNode) mapper.getInput().getSource();
		
		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode());
		
		assertEquals(DataExchangeMode.BATCH, mapperInput.getInput().getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
		
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
示例26
public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
示例27
@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
示例28
public ArchivedExecutionConfig build() {
	return new ArchivedExecutionConfig(
		executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(),
		restartStrategyDescription != null ? restartStrategyDescription : "default",
		parallelism,
		objectReuseEnabled,
		globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap()
	);
}
示例29
@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) {
	Operator<?> children = getOperator().getInput();

	final OptimizerNode pred;
	final DagConnection conn;
	
	pred = contractToNode.get(children);
	conn = new DagConnection(pred, this, defaultExchangeMode);
		
	// create the connection and add it
	this.input = conn;
	pred.addOutgoingConnection(conn);
}
示例30
@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH);
}