private RelNode canonicalize(RelNode rel) {
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(rel);
  return hepPlanner.findBestExp();
}
 

private RelNode decorrelate(RelNode root) {
    // first adjust count() expression if any
    final RelBuilderFactory f = relBuilderFactory();
    HepProgram program = HepProgram.builder().addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
            .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
            .addRuleInstance(new FilterJoinRule.FilterIntoJoinRule(true, f, FilterJoinRule.TRUE_PREDICATE))
            .addRuleInstance(new FilterProjectTransposeRule(Filter.class, Project.class, true, true, f))
            .addRuleInstance(new FilterCorrelateRule(f)).build();

    HepPlanner planner = createPlanner(program);

    planner.setRoot(root);
    root = planner.findBestExp();

    // Perform decorrelation.
    map.clear();

    final Frame frame = getInvoke(root, null);
    if (frame != null) {
        // has been rewritten; apply rules post-decorrelation
        final HepProgram program2 = HepProgram.builder()
                .addRuleInstance(new FilterJoinRule.FilterIntoJoinRule(true, f, FilterJoinRule.TRUE_PREDICATE))
                .addRuleInstance(new FilterJoinRule.JoinConditionPushRule(f, FilterJoinRule.TRUE_PREDICATE))
                .build();

        final HepPlanner planner2 = createPlanner(program2);
        final RelNode newRoot = frame.r;
        planner2.setRoot(newRoot);
        return planner2.findBestExp();
    }

    return root;
}
 

private static List<RelNode> substitute(
    RelNode root, RelOptMaterialization materialization) {
  // First, if the materialization is in terms of a star table, rewrite
  // the query in terms of the star table.
  if (materialization.starTable != null) {
    RelNode newRoot = RelOptMaterialization.tryUseStar(root,
        materialization.starRelOptTable);
    if (newRoot != null) {
      root = newRoot;
    }
  }

  // Push filters to the bottom, and combine projects on top.
  RelNode target = materialization.queryRel;
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .build();

  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(target);
  target = hepPlanner.findBestExp();

  hepPlanner.setRoot(root);
  root = hepPlanner.findBestExp();

  return new MaterializedViewSubstitutionVisitor(target, root)
          .go(materialization.tableRel);
}
 

/** Creates a MaterializedViewAggregateRule. */
protected MaterializedViewAggregateRule(RelOptRuleOperand operand,
    RelBuilderFactory relBuilderFactory, String description,
    boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
  this(operand, relBuilderFactory, description, generateUnionRewriting,
      unionRewritingPullProgram,
      new FilterProjectTransposeRule(
          Filter.class, Project.class, true, true, relBuilderFactory),
      new FilterAggregateTransposeRule(
          Filter.class, relBuilderFactory, Aggregate.class),
      new AggregateProjectPullUpConstantsRule(
          Aggregate.class, Filter.class, relBuilderFactory, "AggFilterPullUpConstants"),
      new ProjectMergeRule(true, ProjectMergeRule.DEFAULT_BLOAT,
          relBuilderFactory));
}
 

@Test void testConvertSemiJoin() {
  final String sql = "select * from dept where exists (\n"
      + "  select * from emp\n"
      + "  where emp.deptno = dept.deptno\n"
      + "  and emp.sal > 100)";
  checkConvertMutableRel(
      "Join", // with join type as semi
      sql,
      true,
      ImmutableList.of(
          FilterProjectTransposeRule.INSTANCE,
          FilterJoinRule.FILTER_ON_JOIN,
          ProjectMergeRule.INSTANCE,
          SemiJoinRule.PROJECT));
}
 

@Test void testHintsPropagationInVolcanoPlannerRules() {
  final String sql = "select /*+ use_hash_join(r, s), use_hash_join(emp, dept) */\n"
      + "ename, job, sal, dept.name\n"
      + "from emp join dept on emp.deptno = dept.deptno";
  RelOptPlanner planner = new VolcanoPlanner();
  planner.addRelTraitDef(ConventionTraitDef.INSTANCE);
  Tester tester1 = tester.withDecorrelation(true)
      .withClusterFactory(
        relOptCluster -> RelOptCluster.create(planner, relOptCluster.getRexBuilder()));
  final RelNode rel = tester1.convertSqlToRel(sql).rel;
  final RelHint hint = RelHint.builder("USE_HASH_JOIN")
      .inheritPath(0)
      .hintOption("EMP")
      .hintOption("DEPT")
      .build();
  // Validate Volcano planner.
  RuleSet ruleSet = RuleSets.ofList(
      new MockEnumerableJoinRule(hint), // Rule to validate the hint.
      FilterProjectTransposeRule.INSTANCE,
      FilterMergeRule.INSTANCE,
      ProjectMergeRule.INSTANCE,
      EnumerableRules.ENUMERABLE_JOIN_RULE,
      EnumerableRules.ENUMERABLE_PROJECT_RULE,
      EnumerableRules.ENUMERABLE_FILTER_RULE,
      EnumerableRules.ENUMERABLE_SORT_RULE,
      EnumerableRules.ENUMERABLE_LIMIT_RULE,
      EnumerableRules.ENUMERABLE_TABLE_SCAN_RULE);
  Program program = Programs.of(ruleSet);
  RelTraitSet toTraits = rel
      .getCluster()
      .traitSet()
      .replace(EnumerableConvention.INSTANCE);

  program.run(planner, rel, toTraits,
      Collections.emptyList(), Collections.emptyList());
}
 

private RelNode decorrelate(RelNode root) {
  // first adjust count() expression if any
  final RelBuilderFactory f = relBuilderFactory();
  HepProgram program = HepProgram.builder()
      .addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
      .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
      .addRuleInstance(
          // use FilterJoinRule instead of FlinkFilterJoinRule while CALCITE-3170 is fixed
          new FlinkFilterJoinRule.FlinkFilterIntoJoinRule(true, f,
              FlinkFilterJoinRule.TRUE_PREDICATE))
      .addRuleInstance(
          new FilterProjectTransposeRule(Filter.class, Project.class, true,
              true, f))
      .addRuleInstance(new FilterCorrelateRule(f))
      .build();

  HepPlanner planner = createPlanner(program);

  planner.setRoot(root);
  root = planner.findBestExp();

  // Perform decorrelation.
  map.clear();

  final Frame frame = getInvoke(root, null);
  if (frame != null) {
    // has been rewritten; apply rules post-decorrelation
    final HepProgram program2 = HepProgram.builder()
        .addRuleInstance(
            // use FilterJoinRule instead of FlinkFilterJoinRule while CALCITE-3170 is fixed
            new FlinkFilterJoinRule.FlinkFilterIntoJoinRule(
                true, f,
                FlinkFilterJoinRule.TRUE_PREDICATE))
        .addRuleInstance(
            new FlinkFilterJoinRule.FlinkJoinConditionPushRule(
                f,
                FlinkFilterJoinRule.TRUE_PREDICATE))
        .build();

    final HepPlanner planner2 = createPlanner(program2);
    final RelNode newRoot = frame.r;
    planner2.setRoot(newRoot);
    return planner2.findBestExp();
  }

  return root;
}
 

private RelNode decorrelate(RelNode root) {
  // first adjust count() expression if any
  final RelBuilderFactory f = relBuilderFactory();
  HepProgram program = HepProgram.builder()
      .addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
      .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
      .addRuleInstance(
          // use FilterJoinRule instead of FlinkFilterJoinRule while CALCITE-3170 is fixed
          new FlinkFilterJoinRule.FlinkFilterIntoJoinRule(true, f,
              FlinkFilterJoinRule.TRUE_PREDICATE))
      .addRuleInstance(
          new FilterProjectTransposeRule(Filter.class, Project.class, true,
              true, f))
      .addRuleInstance(new FilterCorrelateRule(f))
      .build();

  HepPlanner planner = createPlanner(program);

  planner.setRoot(root);
  root = planner.findBestExp();

  // Perform decorrelation.
  map.clear();

  final Frame frame = getInvoke(root, null);
  if (frame != null) {
    // has been rewritten; apply rules post-decorrelation
    final HepProgram program2 = HepProgram.builder()
        .addRuleInstance(
            // use FilterJoinRule instead of FlinkFilterJoinRule while CALCITE-3170 is fixed
            new FlinkFilterJoinRule.FlinkFilterIntoJoinRule(
                true, f,
                FlinkFilterJoinRule.TRUE_PREDICATE))
        .addRuleInstance(
            new FlinkFilterJoinRule.FlinkJoinConditionPushRule(
                f,
                FlinkFilterJoinRule.TRUE_PREDICATE))
        .build();

    final HepPlanner planner2 = createPlanner(program2);
    final RelNode newRoot = frame.r;
    planner2.setRoot(newRoot);
    return planner2.findBestExp();
  }

  return root;
}
 

@Override
public void register(RelOptPlanner planner) {
    // force clear the query context before traversal relational operators
    OLAPContext.clearThreadLocalContexts();

    // register OLAP rules
    addRules(planner, kylinConfig.getCalciteAddRule());

    planner.addRule(OLAPToEnumerableConverterRule.INSTANCE);
    planner.addRule(OLAPFilterRule.INSTANCE);
    planner.addRule(OLAPProjectRule.INSTANCE);
    planner.addRule(OLAPAggregateRule.INSTANCE);
    planner.addRule(OLAPJoinRule.INSTANCE);
    planner.addRule(OLAPLimitRule.INSTANCE);
    planner.addRule(OLAPSortRule.INSTANCE);
    planner.addRule(OLAPUnionRule.INSTANCE);
    planner.addRule(OLAPWindowRule.INSTANCE);
    planner.addRule(OLAPValuesRule.INSTANCE);

    planner.addRule(AggregateProjectReduceRule.INSTANCE);

    // CalcitePrepareImpl.CONSTANT_REDUCTION_RULES
    if (kylinConfig.isReduceExpressionsRulesEnabled()) {
        planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE);
        planner.addRule(ReduceExpressionsRule.FILTER_INSTANCE);
        planner.addRule(ReduceExpressionsRule.CALC_INSTANCE);
        planner.addRule(ReduceExpressionsRule.JOIN_INSTANCE);
    }
    // the ValuesReduceRule breaks query test somehow...
    //        planner.addRule(ValuesReduceRule.FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_INSTANCE);

    removeRules(planner, kylinConfig.getCalciteRemoveRule());
    if (!kylinConfig.isEnumerableRulesEnabled()) {
        for (RelOptRule rule : CalcitePrepareImpl.ENUMERABLE_RULES) {
            planner.removeRule(rule);
        }
    }
    // since join is the entry point, we can't push filter past join
    planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
    planner.removeRule(FilterJoinRule.JOIN);

    // since we don't have statistic of table, the optimization of join is too cost
    planner.removeRule(JoinCommuteRule.INSTANCE);
    planner.removeRule(JoinPushThroughJoinRule.LEFT);
    planner.removeRule(JoinPushThroughJoinRule.RIGHT);

    // keep tree structure like filter -> aggregation -> project -> join/table scan, implementOLAP() rely on this tree pattern
    planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
    planner.removeRule(AggregateProjectMergeRule.INSTANCE);
    planner.removeRule(FilterProjectTransposeRule.INSTANCE);
    planner.removeRule(SortJoinTransposeRule.INSTANCE);
    planner.removeRule(JoinPushExpressionsRule.INSTANCE);
    planner.removeRule(SortUnionTransposeRule.INSTANCE);
    planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
    planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
    planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
    planner.removeRule(DateRangeRules.FILTER_INSTANCE);
    planner.removeRule(SemiJoinRule.JOIN);
    planner.removeRule(SemiJoinRule.PROJECT);
    // distinct count will be split into a separated query that is joined with the left query
    planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);

    // see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser
    planner.removeRule(ExpandConversionRule.INSTANCE);
}
 

@Override
public void register(RelOptPlanner planner) {
    // force clear the query context before traversal relational operators
    OLAPContext.clearThreadLocalContexts();

    // register OLAP rules
    addRules(planner, kylinConfig.getCalciteAddRule());

    planner.addRule(OLAPToEnumerableConverterRule.INSTANCE);
    planner.addRule(OLAPFilterRule.INSTANCE);
    planner.addRule(OLAPProjectRule.INSTANCE);
    planner.addRule(OLAPAggregateRule.INSTANCE);
    planner.addRule(OLAPJoinRule.INSTANCE);
    planner.addRule(OLAPLimitRule.INSTANCE);
    planner.addRule(OLAPSortRule.INSTANCE);
    planner.addRule(OLAPUnionRule.INSTANCE);
    planner.addRule(OLAPWindowRule.INSTANCE);
    planner.addRule(OLAPValuesRule.INSTANCE);

    planner.addRule(AggregateProjectReduceRule.INSTANCE);

    // CalcitePrepareImpl.CONSTANT_REDUCTION_RULES
    if (kylinConfig.isReduceExpressionsRulesEnabled()) {
        planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE);
        planner.addRule(ReduceExpressionsRule.FILTER_INSTANCE);
        planner.addRule(ReduceExpressionsRule.CALC_INSTANCE);
        planner.addRule(ReduceExpressionsRule.JOIN_INSTANCE);
    }
    // the ValuesReduceRule breaks query test somehow...
    //        planner.addRule(ValuesReduceRule.FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_INSTANCE);

    removeRules(planner, kylinConfig.getCalciteRemoveRule());
    if (!kylinConfig.isEnumerableRulesEnabled()) {
        for (RelOptRule rule : CalcitePrepareImpl.ENUMERABLE_RULES) {
            planner.removeRule(rule);
        }
    }
    // since join is the entry point, we can't push filter past join
    planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
    planner.removeRule(FilterJoinRule.JOIN);

    // since we don't have statistic of table, the optimization of join is too cost
    planner.removeRule(JoinCommuteRule.INSTANCE);
    planner.removeRule(JoinPushThroughJoinRule.LEFT);
    planner.removeRule(JoinPushThroughJoinRule.RIGHT);

    // keep tree structure like filter -> aggregation -> project -> join/table scan, implementOLAP() rely on this tree pattern
    planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
    planner.removeRule(AggregateProjectMergeRule.INSTANCE);
    planner.removeRule(FilterProjectTransposeRule.INSTANCE);
    planner.removeRule(SortJoinTransposeRule.INSTANCE);
    planner.removeRule(JoinPushExpressionsRule.INSTANCE);
    planner.removeRule(SortUnionTransposeRule.INSTANCE);
    planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
    planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
    planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
    planner.removeRule(DateRangeRules.FILTER_INSTANCE);
    planner.removeRule(SemiJoinRule.JOIN);
    planner.removeRule(SemiJoinRule.PROJECT);
    // distinct count will be split into a separated query that is joined with the left query
    planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);

    // see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser
    planner.removeRule(ExpandConversionRule.INSTANCE);

    // KYLIN-4464 do not pushdown sort when there is a window function in projection
    planner.removeRule(SortProjectTransposeRule.INSTANCE);
    planner.addRule(KylinSortProjectTransposeRule.INSTANCE);
}
 

protected RelNode decorrelate(RelNode root) {
  // first adjust count() expression if any
  final RelBuilderFactory f = relBuilderFactory();
  HepProgram program = HepProgram.builder()
      .addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
      .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
      .addRuleInstance(
          new FilterJoinRule.FilterIntoJoinRule(true, f,
              FilterJoinRule.TRUE_PREDICATE))
      .addRuleInstance(
          new FilterProjectTransposeRule(Filter.class, Project.class, true,
              true, f))
      .addRuleInstance(new FilterCorrelateRule(f))
      .build();

  HepPlanner planner = createPlanner(program);

  planner.setRoot(root);
  root = planner.findBestExp();

  // Perform decorrelation.
  map.clear();

  final Frame frame = getInvoke(root, null);
  if (frame != null) {
    // has been rewritten; apply rules post-decorrelation
    final HepProgram program2 = HepProgram.builder()
        .addRuleInstance(
            new FilterJoinRule.FilterIntoJoinRule(
                true, f,
                FilterJoinRule.TRUE_PREDICATE))
        .addRuleInstance(
            new FilterJoinRule.JoinConditionPushRule(
                f,
                FilterJoinRule.TRUE_PREDICATE))
        .build();

    final HepPlanner planner2 = createPlanner(program2);
    final RelNode newRoot = frame.r;
    planner2.setRoot(newRoot);
    return planner2.findBestExp();
  }

  return root;
}
 

private static List<RelNode> substitute(
    RelNode root, RelOptMaterialization materialization) {
  // First, if the materialization is in terms of a star table, rewrite
  // the query in terms of the star table.
  if (materialization.starTable != null) {
    RelNode newRoot = RelOptMaterialization.tryUseStar(root,
        materialization.starRelOptTable);
    if (newRoot != null) {
      root = newRoot;
    }
  }

  // Push filters to the bottom, and combine projects on top.
  RelNode target = materialization.queryRel;
  // try to trim unused field in relational expressions.
  root = trimUnusedfields(root);
  target = trimUnusedfields(target);
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();

  // We must use the same HEP planner for the two optimizations below.
  // Thus different nodes with the same digest will share the same vertex in
  // the plan graph. This is important for the matching process.
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(target);
  target = hepPlanner.findBestExp();

  hepPlanner.setRoot(root);
  root = hepPlanner.findBestExp();

  return new SubstitutionVisitor(target, root).go(materialization.tableRel);
}