源码类别
热门源码
 当前位置：源码 > Java源码 >

Java源码示例：org.apache.kylin.metadata.model.ColumnDesc

示例1
@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
示例2
@Override
public void doMap(T key, Object value, Context context) throws IOException, InterruptedException {
    ColumnDesc[] columns = tableDesc.getColumns();
    Collection<String[]> valuesCollection = tableInputFormat.parseMapperInput(value);

    for (String[] values: valuesCollection) {
        for (int m = 0; m < columns.length; m++) {
            String field = columns[m].getName();
            String fieldValue = values[m];
            if (fieldValue == null)
                fieldValue = "NULL";

            if (counter < 5 && m < 10) {
                System.out.println("Get row " + counter + " column '" + field + "'  value: " + fieldValue);
            }

            getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
        }

        counter++;
    }
}
示例3
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
示例4
private List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setTable(t1);
    c1.setDatatype("string");
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c2.setTable(t2);
    c2.setDatatype("string");
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
示例5
protected List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setDatatype("String");
    c1.setTable(t1);
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c1.setDatatype("String");
    c2.setTable(t2);
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
示例6
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        cols[i] = new TblColRef(col);

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes);
}
示例7
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
示例8
private Set<ColumnDesc> findUsedColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    // column in dimension
    for (ModelDimensionDesc dim : usedModel.getDimensions()) {
        if (dim.getTable().equalsIgnoreCase(factTable.getName())) {
            for (String col : dim.getColumns()) {
                usedColumns.add(mustGetColumnDesc(factTable, col));
            }
        }
    }

    // column in measure
    for (String columnInMeasure : usedModel.getMetrics()) {
        if (factTable.getName().equalsIgnoreCase(getTableName(columnInMeasure))) {
            usedColumns.add(mustGetColumnDesc(factTable, columnInMeasure));
        }
    }

    return usedColumns;
}
示例9
private String generateCreateH2TableSql(TableDesc tableDesc, String csvFilePath) {
    StringBuilder ddl = new StringBuilder();
    StringBuilder csvColumns = new StringBuilder();

    ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (col.isComputedColumn()) {
            continue;
        }
        if (i > 0) {
            ddl.append(",");
            csvColumns.append(",");
        }
        ddl.append(col.getName() + " " + getH2DataType((col.getDatatype())) + "\n");
        csvColumns.append(col.getName());
    }
    ddl.append(")" + "\n");
    ddl.append("AS SELECT * FROM CSVREAD('" + csvFilePath + "', '" + csvColumns
            + "', 'charset=UTF-8 fieldSeparator=,');");

    return ddl.toString();
}
示例10
private String[] generateCreateTableSql(TableDesc tableDesc) {
    logger.info("Generate create table sql: {}", tableDesc);
    String tableIdentity = String
            .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName())
            .toUpperCase(Locale.ROOT);
    String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
    String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity;

    StringBuilder ddl = new StringBuilder();
    ddl.append("CREATE TABLE " + tableIdentity + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (i > 0) {
            ddl.append(",");
        }
        ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n");
    }

    ddl.append(")");

    return new String[] { dropsql, dropsql2, ddl.toString() };
}
示例11
/**
 * Used to quote identifiers for JDBC ext job when quoting cc expr
 * @param tableDesc
 * @param sqlExpr
 * @return
 */
public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr, SqlDialect sqlDialect) {
    String table = tableDesc.getName();
    boolean tableMatched = false;
    List<String> tabPatterns = getTableNameOrAliasPatterns(table);
    if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) {
        sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns, sqlDialect);
        tableMatched = true;
    }

    if (tableMatched) {
        for (ColumnDesc columnDesc : tableDesc.getColumns()) {
            String column = columnDesc.getName();
            List<String> colPatterns = getColumnNameOrAliasPatterns(column);
            if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
                sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns, sqlDialect);
            }
        }
    }

    return sqlExpr;
}
示例12
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
示例13
private boolean isColumnCompatible(ColumnDesc column, ColumnDesc newCol) {
    if (!column.getName().equalsIgnoreCase(newCol.getName())) {
        return false;
    }

    if (column.getType().isIntegerFamily()) {
        // OLAPTable.listSourceColumns converts some integer columns to bigint,
        // therefore strict type comparison won't work.
        // changing from one integer type to another should be fine.
        return newCol.getType().isIntegerFamily();
    } else if (column.getType().isNumberFamily()) {
        // Both are float/double should be fine.
        return newCol.getType().isNumberFamily();
    } else {
        // only compare base type name, changing precision or scale should be fine
        return column.getTypeName().equals(newCol.getTypeName());
    }
}
示例14
@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
示例15
private List<String> generateCreateH2IndexSql(TableDesc tableDesc) {
    List<String> result = Lists.newArrayList();
    int x = 0;
    for (ColumnDesc col : tableDesc.getColumns()) {
        if ("T".equalsIgnoreCase(col.getIndex())) {
            StringBuilder ddl = new StringBuilder();
            ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "("
                    + col.getName() + ")");
            ddl.append("\n");
            result.add(ddl.toString());
            x++;
        }
    }

    return result;
}
示例16
@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getUpgradedType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
示例17
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
示例18
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
示例19
private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
    for (TblColRef col : allColumns) {
        if (allColumnDescs.contains(col.getColumnDesc()) == false)
            return false;
    }
    return true;
}
示例20
@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
示例21
@Override
public void createSampleTable(TableDesc table) throws Exception {
    LinkedHashMap<String, String> columnInfo = Maps.newLinkedHashMap();
    for (ColumnDesc columnDesc : table.getColumns()) {
        columnInfo.put(columnDesc.getName(), columnDesc.getTypeName());
    }
    String[] sqls = dataSource.buildSqlToCreateTable(table.getIdentity(), columnInfo);
    dataSource.executeUpdate(sqls);
}
示例22
@Test
public void testSerialize08() {
    TableDesc table = new TableDesc();
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
示例23
public boolean isDefinedColumn(String project, String table, String col) {
    TableCache tableCache = getCache(project).tables.get(table);
    if (tableCache == null)
        return false;

    for (ColumnDesc colDesc : tableCache.tableDesc.getColumns()) {
        if (colDesc.getName().equals(col))
            return true;
    }
    return false;
}
示例24
public Set<ColumnDesc> listExposedColumns(String project, String table) {
    TableCache tableCache = getCache(project).tables.get(table);
    if (tableCache == null)
        return Collections.emptySet();
    else
        return Collections.unmodifiableSet(tableCache.exposedColumns);
}
示例25
@Test
public void testSerialize06() {
    ColumnDesc column = new ColumnDesc();
    column.setName("META_CATEG_NAME");
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
示例26
@Test
public void testSerialize05() {
    ColumnDesc column = new ColumnDesc();

    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
示例27
@Override
protected void init() throws IOException {
    ColumnDesc[] cols = tableDesc.getColumns();
    colIsDateTime = new boolean[cols.length];
    colIsNumber = new boolean[cols.length];
    for (int i = 0; i < cols.length; i++) {
        DataType t = cols[i].getType();
        colIsDateTime[i] = t.isDateTimeFamily();
        colIsNumber[i] = t.isNumberFamily();
    }

    super.init();
}
示例28
private String createCell(ColumnDesc cDesc) throws Exception {
    ColumnConfig cConfig = null;

    if ((cConfig = genConf.getColumnConfigByName(cDesc.getName())) == null) {
        // if the column is not configured, use random values
        return (createRandomCell(cDesc));

    } else {
        // the column has a configuration
        if (!cConfig.isAsRange() && !cConfig.isExclusive() && r.nextBoolean()) {
            // if the column still allows random values
            return (createRandomCell(cDesc));

        } else {
            // use specified values
            ArrayList<String> valueSet = cConfig.getValueSet();
            if (valueSet == null || valueSet.size() == 0)
                throw new Exception("Did you forget to specify value set for " + cDesc.getName());

            if (!cConfig.isAsRange()) {
                return (randomPick(valueSet));
            } else {
                if (valueSet.size() != 2)
                    throw new Exception("Only two values can be set for range values, the column: " + cDesc.getName());

                return (createRandomCell(cDesc, valueSet));
            }
        }

    }
}
示例29
private void init(ColumnDesc col, ModelDataGenerator modelGen) throws IOException {
    
    Map<String, String> config = Util.parseEqualCommaPairs(col.getDataGen(), "values");

    values = Arrays.asList(Util.parseString(config, "values", "").split("[|]"));
    
    List<String> pkValues = modelGen.getPkValuesIfIsFk(col);
    
    if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) {
        isFK = true;
        values = getPkValues(modelGen, config, pkValues);
    } else if (ID.equals(values.get(0))) {
        isID = true;
        idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0;
    } else if (RAND.equals(values.get(0)) || values.get(0).isEmpty()) {
        isRandom = true;
        randFormat = (values.size() > 1) ? values.get(1) : "";
        randStart = (values.size() > 2) ? Integer.parseInt(values.get(2)) : 0;
        randEnd = (values.size() > 3) ? Integer.parseInt(values.get(3)) : 0;
    } else {
        isDiscrete = true;
    }
    
    cardinality = Util.parseInt(config, "card", guessCardinality(col.getName()));
    genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName()));
    genNullPct = Util.parseDouble(config, "nullpct", 0.01);
    genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive
    order = Util.parseBoolean(config, "order", false);
    unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col));
}
示例30
private List<String> getPkValues(ModelDataGenerator modelGen, Map<String, String> config, List<String> dftPkValues) throws IOException {
    String pkColName = config.get("pk");
    if (pkColName == null)
        return dftPkValues;
    
    int cut = pkColName.lastIndexOf('.');
    String pkTableName = pkColName.substring(0, cut);
    pkColName = pkColName.substring(cut + 1);
    
    KylinConfig kylinConfig = modelGen.getModle().getConfig();
    String project = modelGen.getModle().getProject();
    ColumnDesc pkcol = TableMetadataManager.getInstance(kylinConfig)//
            .getTableDesc(pkTableName, project).findColumnByName(pkColName);
    return modelGen.getPkValues(pkcol);
}