Java源码示例:org.apache.lucene.index.DocValuesType
示例1
static CodecReader wrap(CodecReader reader) throws IOException {
final FieldInfos fieldInfos = reader.getFieldInfos();
final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME);
if (versionInfo != null && versionInfo.getDocValuesType() != DocValuesType.NONE) {
// the reader is a recent one, it has versions and they are stored
// in a numeric doc values field
return reader;
}
// The segment is an old one, look at the _uid field
final Terms terms = reader.terms(UidFieldMapper.NAME);
if (terms == null || !terms.hasPayloads()) {
// The segment doesn't have an _uid field or doesn't have payloads
// don't try to do anything clever. If any other segment has versions
// all versions of this segment will be initialized to 0
return reader;
}
// convert _uid payloads -> _version docvalues
return new VersionFieldUpgrader(reader);
}
示例2
@Override
public ParentFieldMapper build(BuilderContext context) {
if (parentType == null) {
throw new MapperParsingException("[_parent] field mapping must contain the [type] option");
}
parentJoinFieldType.setNames(new MappedFieldType.Names(joinField(documentType)));
parentJoinFieldType.setFieldDataType(null);
childJoinFieldType.setNames(new MappedFieldType.Names(joinField(parentType)));
if (context.indexCreatedVersion().before(Version.V_2_0_0_beta1)) {
childJoinFieldType.setHasDocValues(false);
childJoinFieldType.setDocValuesType(DocValuesType.NONE);
parentJoinFieldType.setHasDocValues(false);
parentJoinFieldType.setDocValuesType(DocValuesType.NONE);
}
return new ParentFieldMapper(fieldType, parentJoinFieldType, childJoinFieldType, parentType, context.indexSettings());
}
示例3
public void testIndexingPointsAndDocValues() throws Exception {
FieldType type = new FieldType();
type.setDimensions(1, 4);
type.setDocValuesType(DocValuesType.BINARY);
type.freeze();
Document doc = new Document();
byte[] packedPoint = "term".getBytes(StandardCharsets.UTF_8);
doc.add(new BinaryPoint("field", packedPoint, type));
MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
assertEquals(1, leafReader.getPointValues("field").size());
assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMinPackedValue());
assertArrayEquals(packedPoint, leafReader.getPointValues("field").getMaxPackedValue());
BinaryDocValues dvs = leafReader.getBinaryDocValues("field");
assertEquals(0, dvs.nextDoc());
assertEquals("term", dvs.binaryValue().utf8ToString());
}
示例4
public void testToStringDebug() {
MemoryIndex mi = new MemoryIndex(true, true);
Analyzer analyzer = new MockPayloadAnalyzer();
mi.addField("analyzedField", "aa bb aa", analyzer);
FieldType type = new FieldType();
type.setDimensions(1, 4);
type.setDocValuesType(DocValuesType.BINARY);
type.freeze();
mi.addField(new BinaryPoint("pointAndDvField", "term".getBytes(StandardCharsets.UTF_8), type), analyzer);
assertEquals("analyzedField:\n" +
"\t'[61 61]':2: [(0, 0, 2, [70 6f 73 3a 20 30]), (1, 6, 8, [70 6f 73 3a 20 32])]\n" +
"\t'[62 62]':1: [(1, 3, 5, [70 6f 73 3a 20 31])]\n" +
"\tterms=2, positions=3\n" +
"pointAndDvField:\n" +
"\tterms=0, positions=0\n" +
"\n" +
"fields=2, terms=2, positions=3", mi.toStringDebug());
}
示例5
/**
* Create a new instance configured with the provided FieldType options. See {@link #DEFAULT_FIELDTYPE}.
* a field type is used to articulate the desired options (namely pointValues, docValues, stored). Legacy numerics
* is configurable this way too.
*/
public PointVectorStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
super(ctx, fieldNamePrefix);
this.fieldNameX = fieldNamePrefix+SUFFIX_X;
this.fieldNameY = fieldNamePrefix+SUFFIX_Y;
int numPairs = 0;
if ((this.hasStored = fieldType.stored())) {
numPairs++;
}
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numPairs++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
numPairs++;
}
this.fieldsLen = numPairs * 2;
}
示例6
/**
* Returns the doc values for the specified field in the specified document.
* Empty Optional instance is returned if no doc values is available for the field.
*
* @param docid - document id
* @param field - field name
* @return doc values, if exists, or empty
* @throws IOException - if there is a low level IO error.
*/
Optional<DocValues> getDocValues(int docid, String field) throws IOException {
DocValuesType dvType = IndexUtils.getFieldInfo(reader, field).getDocValuesType();
switch (dvType) {
case BINARY:
return createBinaryDocValues(docid, field, DocValuesType.BINARY);
case NUMERIC:
return createNumericDocValues(docid, field, DocValuesType.NUMERIC);
case SORTED_NUMERIC:
return createSortedNumericDocValues(docid, field, DocValuesType.SORTED_NUMERIC);
case SORTED:
return createSortedDocValues(docid, field, DocValuesType.SORTED);
case SORTED_SET:
return createSortedSetDocValues(docid, field, DocValuesType.SORTED_SET);
default:
return Optional.empty();
}
}
示例7
private Optional<DocValues> createSortedNumericDocValues(int docid, String field, DocValuesType dvType)
throws IOException {
SortedNumericDocValues snvalues = IndexUtils.getSortedNumericDocValues(reader, field);
if (snvalues.advanceExact(docid)) {
List<Long> numericValues = new ArrayList<>();
int dvCount = snvalues.docValueCount();
for (int i = 0; i < dvCount; i++) {
numericValues.add(snvalues.nextValue());
}
DocValues dv = DocValues.of(
dvType,
Collections.emptyList(),
numericValues
);
return Optional.of(dv);
}
return Optional.empty();
}
示例8
private Optional<DocValues> createSortedSetDocValues(int docid, String field, DocValuesType dvType)
throws IOException {
SortedSetDocValues ssvalues = IndexUtils.getSortedSetDocvalues(reader, field);
if (ssvalues.advanceExact(docid)) {
List<BytesRef> values = new ArrayList<>();
long ord;
while ((ord = ssvalues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
values.add(BytesRef.deepCopyOf(ssvalues.lookupOrd(ord)));
}
DocValues dv = DocValues.of(
dvType,
values,
Collections.emptyList()
);
return Optional.of(dv);
}
return Optional.empty();
}
示例9
private static FieldInfo getMockFieldInfo(String fieldName, int number) {
return new FieldInfo(fieldName,
number,
false,
false,
true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE,
-1,
Collections.emptyMap(),
0,
0,
0,
true
);
}
示例10
private static FieldInfo mockFieldInfo(String fieldName, int number) {
return new FieldInfo(fieldName,
number,
false,
false,
true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
DocValuesType.NONE,
-1,
Collections.emptyMap(),
0,
0,
0,
false
);
}
示例11
private static byte docValuesByte(DocValuesType type) {
switch(type) {
case NONE:
return 0;
case NUMERIC:
return 1;
case BINARY:
return 2;
case SORTED:
return 3;
case SORTED_SET:
return 4;
case SORTED_NUMERIC:
return 5;
default:
// BUG
throw new AssertionError("unhandled DocValuesType: " + type);
}
}
示例12
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
switch(b) {
case 0:
return DocValuesType.NONE;
case 1:
return DocValuesType.NUMERIC;
case 2:
return DocValuesType.BINARY;
case 3:
return DocValuesType.SORTED;
case 4:
return DocValuesType.SORTED_SET;
case 5:
return DocValuesType.SORTED_NUMERIC;
default:
throw new CorruptIndexException("invalid docvalues byte: " + b, input);
}
}
示例13
private static byte docValuesByte(DocValuesType type) {
switch(type) {
case NONE:
return 0;
case NUMERIC:
return 1;
case BINARY:
return 2;
case SORTED:
return 3;
case SORTED_SET:
return 4;
case SORTED_NUMERIC:
return 5;
default:
// BUG
throw new AssertionError("unhandled DocValuesType: " + type);
}
}
示例14
private static DocValuesType getDocValuesType(IndexInput input, byte b) throws IOException {
switch(b) {
case 0:
return DocValuesType.NONE;
case 1:
return DocValuesType.NUMERIC;
case 2:
return DocValuesType.BINARY;
case 3:
return DocValuesType.SORTED;
case 4:
return DocValuesType.SORTED_SET;
case 5:
return DocValuesType.SORTED_NUMERIC;
default:
throw new CorruptIndexException("invalid docvalues byte: " + b, input);
}
}
示例15
@Override
public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException {
final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
if (fieldInfo == null) {
// field does not exist or has no value
return new Bits.MatchNoBits(reader.maxDoc());
}
if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
// doc values case
} else if (parser instanceof PointParser) {
// points case
} else {
// postings case
if (fieldInfo.getIndexOptions() == IndexOptions.NONE) {
return new Bits.MatchNoBits(reader.maxDoc());
}
}
BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser));
return bitsEntry.bits;
}
示例16
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
SortedDocValues valuesIn = reader.getSortedDocValues(field);
if (valuesIn != null) {
// Not cached here by FieldCacheImpl (cached instead
// per-thread by SegmentReader):
return valuesIn;
} else {
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.emptySorted();
} else if (info.getDocValuesType() != DocValuesType.NONE) {
// we don't try to build a sorted instance from numeric/binary doc
// values because dedup can be very costly
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (info.getIndexOptions() == IndexOptions.NONE) {
return DocValues.emptySorted();
}
SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
return impl.iterator();
}
}
示例17
public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
if (valuesIn == null) {
valuesIn = reader.getSortedDocValues(field);
}
if (valuesIn != null) {
// Not cached here by FieldCacheImpl (cached instead
// per-thread by SegmentReader):
return valuesIn;
}
final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
if (info == null) {
return DocValues.emptyBinary();
} else if (info.getDocValuesType() != DocValuesType.NONE) {
throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
} else if (info.getIndexOptions() == IndexOptions.NONE) {
return DocValues.emptyBinary();
}
BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
return impl.iterator();
}
示例18
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) {
super(fcontext, freq, sf);
if (freq.mincount == 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" doesn't support mincount=0");
}
if (freq.prefix != null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" doesn't support prefix"); // yet, but it could
}
FieldInfo fieldInfo = fcontext.searcher.getFieldInfos().fieldInfo(sf.getName());
if (fieldInfo != null &&
fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
fieldInfo.getDocValuesType() != DocValuesType.SORTED &&
fieldInfo.getDocValuesType() != DocValuesType.SORTED_NUMERIC) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" only support single valued number/string with docValues");
}
}
示例19
VersionFieldUpgrader(CodecReader in) {
super(in);
// Find a free field number
int fieldNumber = 0;
for (FieldInfo fi : in.getFieldInfos()) {
fieldNumber = Math.max(fieldNumber, fi.number + 1);
}
// TODO: lots of things can wrong here...
FieldInfo newInfo = new FieldInfo(VersionFieldMapper.NAME, // field name
fieldNumber, // field number
false, // store term vectors
false, // omit norms
false, // store payloads
IndexOptions.NONE, // index options
DocValuesType.NUMERIC, // docvalues
-1, // docvalues generation
Collections.<String, String>emptyMap() // attributes
);
newInfo.checkConsistency(); // fail merge immediately if above code is wrong
final ArrayList<FieldInfo> fieldInfoList = new ArrayList<>();
for (FieldInfo info : in.getFieldInfos()) {
if (!info.name.equals(VersionFieldMapper.NAME)) {
fieldInfoList.add(info);
}
}
fieldInfoList.add(newInfo);
infos = new FieldInfos(fieldInfoList.toArray(new FieldInfo[fieldInfoList.size()]));
}
示例20
private static MappedFieldType joinFieldTypeForParentType(String parentType, Settings indexSettings) {
MappedFieldType parentJoinFieldType = Defaults.JOIN_FIELD_TYPE.clone();
parentJoinFieldType.setNames(new MappedFieldType.Names(joinField(parentType)));
Version indexCreated = Version.indexCreated(indexSettings);
if (indexCreated.before(Version.V_2_0_0_beta1)) {
parentJoinFieldType.setHasDocValues(false);
parentJoinFieldType.setDocValuesType(DocValuesType.NONE);
}
parentJoinFieldType.freeze();
return parentJoinFieldType;
}
示例21
public TermVectorLeafReader(String field, Terms terms) {
fields = new Fields() {
@Override
public Iterator<String> iterator() {
return Collections.singletonList(field).iterator();
}
@Override
public Terms terms(String fld) throws IOException {
if (!field.equals(fld)) {
return null;
}
return terms;
}
@Override
public int size() {
return 1;
}
};
IndexOptions indexOptions;
if (!terms.hasFreqs()) {
indexOptions = IndexOptions.DOCS;
} else if (!terms.hasPositions()) {
indexOptions = IndexOptions.DOCS_AND_FREQS;
} else if (!terms.hasOffsets()) {
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
} else {
indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
}
FieldInfo fieldInfo = new FieldInfo(field, 0,
true, true, terms.hasPayloads(),
indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}
示例22
/** helper: checks a fieldinfo and throws exception if its definitely not a Geo3DDocValuesField */
static void checkCompatible(FieldInfo fieldInfo) {
// dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) {
throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() +
" but this type has docValuesType=" + TYPE.docValuesType() +
", is the field really a Geo3DDocValuesField?");
}
}
示例23
/**
* Creates this strategy.
* {@code fieldType} is used to customize the indexing options of the 4 number fields, and to a lesser degree the XDL
* field too. Search requires pointValues (or legacy numerics), and relevancy requires docValues. If these features
* aren't needed then disable them.
*/
public BBoxStrategy(SpatialContext ctx, String fieldNamePrefix, FieldType fieldType) {
super(ctx, fieldNamePrefix);
field_bbox = fieldNamePrefix;
field_minX = fieldNamePrefix + SUFFIX_MINX;
field_maxX = fieldNamePrefix + SUFFIX_MAXX;
field_minY = fieldNamePrefix + SUFFIX_MINY;
field_maxY = fieldNamePrefix + SUFFIX_MAXY;
field_xdl = fieldNamePrefix + SUFFIX_XDL;
fieldType.freeze();
this.optionsFieldType = fieldType;
int numQuads = 0;
if ((this.hasStored = fieldType.stored())) {
numQuads++;
}
if ((this.hasDocVals = fieldType.docValuesType() != DocValuesType.NONE)) {
numQuads++;
}
if ((this.hasPointVals = fieldType.pointDimensionCount() > 0)) {
numQuads++;
}
if (hasPointVals) { // if we have an index...
xdlFieldType = new FieldType(StringField.TYPE_NOT_STORED);
xdlFieldType.setIndexOptions(IndexOptions.DOCS);
xdlFieldType.freeze();
} else {
xdlFieldType = null;
}
this.fieldsLen = numQuads * 4 + (xdlFieldType != null ? 1 : 0);
}
示例24
@Test
public void testOperations() throws IOException {
//setup
if (random().nextInt(4) > 0) {//75% of the time choose geo (more interesting to test)
this.ctx = SpatialContext.GEO;
} else {
SpatialContextFactory factory = new SpatialContextFactory();
factory.geo = false;
factory.worldBounds = new RectangleImpl(-300, 300, -100, 100, null);
this.ctx = factory.newSpatialContext();
}
this.strategy = BBoxStrategy.newInstance(ctx, "bbox");
//test we can disable docValues for predicate tests
if (random().nextBoolean()) {
FieldType fieldType = new FieldType(((BBoxStrategy)strategy).getFieldType());
fieldType.setDocValuesType(DocValuesType.NONE);
strategy = new BBoxStrategy(ctx, strategy.getFieldName(), fieldType);
}
for (SpatialOperation operation : SpatialOperation.values()) {
if (operation == SpatialOperation.Overlaps)
continue;//unsupported
testOperationRandomShapes(operation);
deleteAll();
commit();
}
}
示例25
@Override
public Collection<String> getSortableFieldNames() {
return IndexUtils.getFieldNames(reader).stream()
.map(f -> IndexUtils.getFieldInfo(reader, f))
.filter(info -> !info.getDocValuesType().equals(DocValuesType.NONE))
.map(info -> info.name)
.collect(Collectors.toList());
}
示例26
private Optional<DocValues> createBinaryDocValues(int docid, String field, DocValuesType dvType)
throws IOException {
BinaryDocValues bvalues = IndexUtils.getBinaryDocValues(reader, field);
if (bvalues.advanceExact(docid)) {
DocValues dv = DocValues.of(
dvType,
Collections.singletonList(BytesRef.deepCopyOf(bvalues.binaryValue())),
Collections.emptyList());
return Optional.of(dv);
}
return Optional.empty();
}
示例27
private Optional<DocValues> createNumericDocValues(int docid, String field, DocValuesType dvType)
throws IOException{
NumericDocValues nvalues = IndexUtils.getNumericDocValues(reader, field);
if (nvalues.advanceExact(docid)) {
DocValues dv = DocValues.of(
dvType,
Collections.emptyList(),
Collections.singletonList(nvalues.longValue())
);
return Optional.of(dv);
}
return Optional.empty();
}
示例28
private Optional<DocValues> createSortedDocValues(int docid, String field, DocValuesType dvType)
throws IOException {
SortedDocValues svalues = IndexUtils.getSortedDocValues(reader, field);
if (svalues.advanceExact(docid)) {
DocValues dv = DocValues.of(
dvType,
Collections.singletonList(BytesRef.deepCopyOf(svalues.binaryValue())),
Collections.emptyList()
);
return Optional.of(dv);
}
return Optional.empty();
}
示例29
@Test
public void testGetDocValues_binary() throws Exception {
DocValuesAdapter adapterImpl = new DocValuesAdapter(reader);
DocValues values = adapterImpl.getDocValues(0, "dv_binary").orElseThrow(IllegalStateException::new);
assertEquals(DocValuesType.BINARY, values.getDvType());
assertEquals(new BytesRef("lucene"), values.getValues().get(0));
assertEquals(Collections.emptyList(), values.getNumericValues());
}
示例30
@Test
public void testGetDocValues_sorted() throws Exception {
DocValuesAdapter adapterImpl = new DocValuesAdapter(reader);
DocValues values = adapterImpl.getDocValues(0, "dv_sorted").orElseThrow(IllegalStateException::new);
assertEquals(DocValuesType.SORTED, values.getDvType());
assertEquals(new BytesRef("abc"), values.getValues().get(0));
assertEquals(Collections.emptyList(), values.getNumericValues());
}