Java源码示例:org.apache.spark.sql.sources.v2.reader.DataReader
示例1
@Override
public DataReader<ColumnarBatch> createDataReader() {
LlapInputSplit llapInputSplit = new LlapInputSplit();
ByteArrayInputStream splitByteArrayStream = new ByteArrayInputStream(splitBytes);
ByteArrayInputStream confByteArrayStream = new ByteArrayInputStream(confBytes);
JobConf conf = new JobConf();
try(DataInputStream splitByteData = new DataInputStream(splitByteArrayStream);
DataInputStream confByteData = new DataInputStream(confByteArrayStream)) {
llapInputSplit.readFields(splitByteData);
conf.readFields(confByteData);
return getDataReader(llapInputSplit, conf, arrowAllocatorMax);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
示例2
@Override
public DataReader<Row> createDataReader() {
log.info("Factory creating reader for [" + _host + ":" + _port + "]" );
try {
return new TaskDataReader(_host, _port);
} catch (UnknownTableException ute) {
throw new RuntimeException(ute);
}
}
示例3
@Override
public DataReader<Row> createDataReader() {
log.info("Factory creating reader for [" + _host + ":" + _port + "]" );
try {
return new TaskDataReader(_host, _port, _table, _schema, _split);
} catch (UnknownTableException ute) {
throw new RuntimeException(ute);
}
}
示例4
@Override
public DataReader<Row> createDataReader() {
log.info("Factory creating reader for [" + _host + ":" + _port + "]" );
try {
return new TaskDataReader(_host, _port, _table, _schema);
} catch (UnknownTableException ute) {
throw new RuntimeException(ute);
}
}
示例5
@Override
public DataReader<Row> createDataReader() {
log.info("Factory creating reader for [" + _host + ":" + _port + "]" );
try {
return new TaskDataReader(_host, _port, _table, _schema, _split);
} catch (UnknownTableException ute) {
throw new RuntimeException(ute);
}
}
示例6
@Override
public DataReader<Row> createDataReader() {
log.info("Factory creating reader for [" + _host + ":" + _port + "]" );
try {
return new TaskDataReader(_host, _port, _table, _schema, _split);
} catch (UnknownTableException ute) {
throw new RuntimeException(ute);
}
}
示例7
@Override
public DataReader<ColumnarBatch> createDataReader() {
try {
return getDataReader(null, new JobConf(), Long.MAX_VALUE);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
示例8
@Override
public DataReader<UnsafeRow> createDataReader() {
return new TaskDataReader(task, lazyTableSchema(), lazyExpectedSchema(), conf.value());
}
示例9
@Override
public DataReader<ColumnarBatch> createDataReader() {
return new CountDataReader(numRows);
}
示例10
protected DataReader<ColumnarBatch> getDataReader(LlapInputSplit split, JobConf jobConf, long arrowAllocatorMax)
throws Exception {
return new HiveWarehouseDataReader(split, jobConf, arrowAllocatorMax);
}
示例11
@Override
public DataReader<Row> createDataReader() {
return new SimpleMockDataReader();
}
示例12
@Override
protected DataReader<ColumnarBatch> getDataReader(LlapInputSplit split, JobConf jobConf, long arrowAllocatorMax)
throws Exception {
return new MockHiveWarehouseDataReader(split, jobConf, arrowAllocatorMax);
}