diff options
author | Jim Ferenczi <jim.ferenczi@elastic.co> | 2017-05-19 17:11:23 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-05-19 17:11:23 +0200 |
commit | d241c4898e5d5ab87284cc3b351989d26947b552 (patch) | |
tree | 2aefac4d68554a2a4d0de3d26f8e79414858f14f /core | |
parent | b18df27d74664ac3254b8395b67d097b487fd7ce (diff) |
Removes parent child fielddata specialization (#24737)
This change removes the field data specialization needed for the parent field and replaces it with
a simple DocValuesIndexFieldData. The underlying global ordinals are retrieved via a new function called
IndexOrdinalsFieldData#getOrdinalMap.
The children aggregation is also modified to use a simple WithOrdinals value source rather than the deleted WithOrdinals.Parent.
Relates #20257
Diffstat (limited to 'core')
16 files changed, 104 insertions, 1044 deletions
diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java deleted file mode 100644 index f88d7c5877..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/AtomicParentChildFieldData.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.index.SortedDocValues; - -import java.util.Set; - -/** - * Specialization of {@link AtomicFieldData} for parent/child mappings. - */ -public interface AtomicParentChildFieldData extends AtomicFieldData { - - /** - * Return the set of types there is a mapping for. - */ - Set<String> types(); - - /** - * Return the mapping for the given type. The returned - * {@link SortedDocValues} will map doc IDs to the identifier of their - * parent. - */ - SortedDocValues getOrdinalsValues(String type); - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java index cb1471179c..2e714fc80a 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalsFieldData.java @@ -21,7 +21,7 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; - +import org.apache.lucene.index.MultiDocValues; /** @@ -42,4 +42,9 @@ public interface IndexOrdinalsFieldData extends IndexFieldData.Global<AtomicOrdi @Override IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception; + /** + * Returns the underlying {@link MultiDocValues.OrdinalMap} for this fielddata + * or null if global ordinals are not needed (constant value or single segment). + */ + MultiDocValues.OrdinalMap getOrdinalMap(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java deleted file mode 100644 index 0453b3f1a1..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexParentChildFieldData.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; - - - - -/** - * Soecialization of {@link IndexFieldData} for parent/child mappings. - */ -public interface IndexParentChildFieldData extends IndexFieldData.Global<AtomicParentChildFieldData> { - - /** - * Load a global view of the ordinals for the given {@link IndexReader}, - * potentially from a cache. - */ - @Override - IndexParentChildFieldData loadGlobal(DirectoryReader indexReader); - - /** - * Load a global view of the ordinals for the given {@link IndexReader}. - */ - @Override - IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception; - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java index e8dea836e3..d0e8285be2 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; @@ -99,6 +100,11 @@ public final class UidIndexFieldData implements IndexOrdinalsFieldData { return new UidIndexFieldData(index, type, idFieldData.localGlobalDirect(indexReader)); } + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return idFieldData.getOrdinalMap(); + } + static final class UidAtomicFieldData implements AtomicOrdinalsFieldData { private final BytesRef prefix; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java index 49140968ca..0a8ab3ccd4 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsBuilder.java @@ -74,7 +74,7 @@ public enum GlobalOrdinalsBuilder { new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS) ); } - return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), + return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, memorySizeInBytes, scriptFunction ); } @@ -108,7 +108,7 @@ public enum GlobalOrdinalsBuilder { subs[i] = atomicFD[i].getOrdinalsValues(); } final OrdinalMap ordinalMap = OrdinalMap.build(null, subs, PackedInts.DEFAULT); - return new InternalGlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), + return new GlobalOrdinalsIndexFieldData(indexSettings, indexFieldData.getFieldName(), atomicFD, ordinalMap, 0, AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION ); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java index 2055208021..795e4b9920 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/GlobalOrdinalsIndexFieldData.java @@ -20,6 +20,8 @@ package org.elasticsearch.index.fielddata.ordinals; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.Nullable; @@ -29,23 +31,39 @@ import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; import org.elasticsearch.search.MultiValueMode; import java.util.Collection; import java.util.Collections; +import java.util.function.Function; /** * {@link IndexFieldData} base class for concrete global ordinals implementations. */ -public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable { +public class GlobalOrdinalsIndexFieldData extends AbstractIndexComponent implements IndexOrdinalsFieldData, Accountable { private final String fieldName; private final long memorySizeInBytes; - protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, long memorySizeInBytes) { + private final MultiDocValues.OrdinalMap ordinalMap; + private final Atomic[] atomicReaders; + private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction; + + + protected GlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, + MultiDocValues.OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues, + ScriptDocValues<?>> scriptFunction) { super(indexSettings); this.fieldName = fieldName; this.memorySizeInBytes = memorySizeInBytes; + this.ordinalMap = ordinalMap; + this.atomicReaders = new Atomic[segmentAfd.length]; + for (int i = 0; i < segmentAfd.length; i++) { + atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i); + } + this.scriptFunction = scriptFunction; } @Override @@ -88,4 +106,57 @@ public abstract class GlobalOrdinalsIndexFieldData extends AbstractIndexComponen // TODO: break down ram usage? return Collections.emptyList(); } + + @Override + public AtomicOrdinalsFieldData load(LeafReaderContext context) { + return atomicReaders[context.ord]; + } + + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return ordinalMap; + } + + private final class Atomic extends AbstractAtomicOrdinalsFieldData { + + private final AtomicOrdinalsFieldData afd; + private final MultiDocValues.OrdinalMap ordinalMap; + private final int segmentIndex; + + private Atomic(AtomicOrdinalsFieldData afd, MultiDocValues.OrdinalMap ordinalMap, int segmentIndex) { + super(scriptFunction); + this.afd = afd; + this.ordinalMap = ordinalMap; + this.segmentIndex = segmentIndex; + } + + @Override + public SortedSetDocValues getOrdinalsValues() { + final SortedSetDocValues values = afd.getOrdinalsValues(); + if (values.getValueCount() == ordinalMap.getValueCount()) { + // segment ordinals match global ordinals + return values; + } + final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; + for (int i = 0; i < bytesValues.length; i++) { + bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); + } + return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex); + } + + @Override + public long ramBytesUsed() { + return afd.ramBytesUsed(); + } + + @Override + public Collection<Accountable> getChildResources() { + return afd.getChildResources(); + } + + @Override + public void close() { + } + + } } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java deleted file mode 100644 index 23ecc06fed..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/ordinals/InternalGlobalOrdinalsIndexFieldData.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata.ordinals; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData; - -import java.util.Collection; -import java.util.function.Function; - -/** - * {@link org.elasticsearch.index.fielddata.IndexFieldData} impl based on global ordinals. - */ -final class InternalGlobalOrdinalsIndexFieldData extends GlobalOrdinalsIndexFieldData { - - private final Atomic[] atomicReaders; - private final Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction; - - InternalGlobalOrdinalsIndexFieldData(IndexSettings indexSettings, String fieldName, AtomicOrdinalsFieldData[] segmentAfd, - OrdinalMap ordinalMap, long memorySizeInBytes, Function<SortedSetDocValues, ScriptDocValues<?>> scriptFunction) { - super(indexSettings, fieldName, memorySizeInBytes); - this.atomicReaders = new Atomic[segmentAfd.length]; - for (int i = 0; i < segmentAfd.length; i++) { - atomicReaders[i] = new Atomic(segmentAfd[i], ordinalMap, i); - } - this.scriptFunction = scriptFunction; - } - - @Override - public AtomicOrdinalsFieldData load(LeafReaderContext context) { - return atomicReaders[context.ord]; - } - - private final class Atomic extends AbstractAtomicOrdinalsFieldData { - - private final AtomicOrdinalsFieldData afd; - private final OrdinalMap ordinalMap; - private final int segmentIndex; - - private Atomic(AtomicOrdinalsFieldData afd, OrdinalMap ordinalMap, int segmentIndex) { - super(scriptFunction); - this.afd = afd; - this.ordinalMap = ordinalMap; - this.segmentIndex = segmentIndex; - } - - @Override - public SortedSetDocValues getOrdinalsValues() { - final SortedSetDocValues values = afd.getOrdinalsValues(); - if (values.getValueCount() == ordinalMap.getValueCount()) { - // segment ordinals match global ordinals - return values; - } - final SortedSetDocValues[] bytesValues = new SortedSetDocValues[atomicReaders.length]; - for (int i = 0; i < bytesValues.length; i++) { - bytesValues[i] = atomicReaders[i].afd.getOrdinalsValues(); - } - return new GlobalOrdinalMapping(ordinalMap, bytesValues, segmentIndex); - } - - @Override - public long ramBytesUsed() { - return afd.ramBytesUsed(); - } - - @Override - public Collection<Accountable> getChildResources() { - return afd.getChildResources(); - } - - @Override - public void close() { - } - - } - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java deleted file mode 100644 index 2df5aa6bb6..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicParentChildFieldData.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - -import static java.util.Collections.emptySet; - - -abstract class AbstractAtomicParentChildFieldData implements AtomicParentChildFieldData { - - @Override - public final ScriptDocValues getScriptValues() { - return new ScriptDocValues.Strings(getBytesValues()); - } - - @Override - public final SortedBinaryDocValues getBytesValues() { - return new SortedBinaryDocValues() { - - private final SortedDocValues[] perTypeValues; - private final BytesRef[] terms = new BytesRef[2]; - private int count; - private int termsCursor; - - { - Set<String> types = types(); - perTypeValues = new SortedDocValues[types.size()]; - int i = 0; - for (String type : types) { - perTypeValues[i++] = getOrdinalsValues(type); - } - } - - @Override - public boolean advanceExact(int docId) throws IOException { - count = 0; - termsCursor = 0; - - for (SortedDocValues values : perTypeValues) { - if (values.advanceExact(docId)) { - final int ord = values.ordValue(); - terms[count++] = values.lookupOrd(ord); - } - } - assert count <= 2 : "A single doc can potentially be both parent and child, so the maximum allowed values is 2"; - if (count > 1) { - int cmp = terms[0].compareTo(terms[1]); - if (cmp > 0) { - ArrayUtil.swap(terms, 0, 1); - } else if (cmp == 0) { - // If the id is the same between types the only omit one. For example: a doc has parent#1 in _uid field and has grand_parent#1 in _parent field. - count = 1; - } - } - return count != 0; - } - - @Override - public int docValueCount() { - return count; - } - - @Override - public BytesRef nextValue() throws IOException { - return terms[termsCursor++]; - } - }; - } - - public static AtomicParentChildFieldData empty() { - return new AbstractAtomicParentChildFieldData() { - - @Override - public long ramBytesUsed() { - return 0; - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - return DocValues.emptySorted(); - } - - @Override - public Set<String> types() { - return emptySet(); - } - }; - } -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java index ce05e226b7..1dbd082f93 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractIndexOrdinalsFieldData.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -52,6 +53,11 @@ public abstract class AbstractIndexOrdinalsFieldData extends AbstractIndexFieldD } @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return null; + } + + @Override public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) { if (indexReader.leaves().size() <= 1) { // ordinals are already global diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java deleted file mode 100644 index 74b180f2c1..0000000000 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ /dev/null @@ -1,401 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata.plain; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.index.MultiDocValues.OrdinalMap; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LongValues; -import org.apache.lucene.util.packed.PackedInts; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.lease.Releasable; -import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AbstractSortedDocValues; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParentFieldMapper; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.search.MultiValueMode; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * ParentChildIndexFieldData is responsible for loading the id cache mapping - * needed for has_child and has_parent queries into memory. - */ -public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicParentChildFieldData> implements IndexParentChildFieldData { - - private final Set<String> parentTypes; - private final CircuitBreakerService breakerService; - - public ParentChildIndexFieldData(IndexSettings indexSettings, String fieldName, - IndexFieldDataCache cache, MapperService mapperService, - CircuitBreakerService breakerService) { - super(indexSettings, fieldName, cache); - this.breakerService = breakerService; - Set<String> parentTypes = new HashSet<>(); - for (DocumentMapper mapper : mapperService.docMappers(false)) { - ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper(); - if (parentFieldMapper.active()) { - parentTypes.add(parentFieldMapper.type()); - } - } - this.parentTypes = parentTypes; - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); - return new SortField(getFieldName(), source, reverse); - } - - @Override - public AtomicParentChildFieldData load(LeafReaderContext context) { - final LeafReader reader = context.reader(); - return new AbstractAtomicParentChildFieldData() { - - public Set<String> types() { - return parentTypes; - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - try { - return DocValues.getSorted(reader, ParentFieldMapper.joinField(type)); - } catch (IOException e) { - throw new IllegalStateException("cannot load join doc values field for type [" + type + "]", e); - } - } - - @Override - public long ramBytesUsed() { - // unknown - return 0; - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() throws ElasticsearchException { - } - }; - } - - @Override - public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - protected AtomicParentChildFieldData empty(int maxDoc) { - return AbstractAtomicParentChildFieldData.empty(); - } - - public static class Builder implements IndexFieldData.Builder { - - @Override - public IndexFieldData<?> build(IndexSettings indexSettings, - MappedFieldType fieldType, - IndexFieldDataCache cache, CircuitBreakerService breakerService, - MapperService mapperService) { - return new ParentChildIndexFieldData(indexSettings, fieldType.name(), cache, - mapperService, breakerService); - } - } - - @Override - public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.leaves().size() <= 1) { - // ordinals are already global - return this; - } - try { - return cache.load(indexReader, this); - } catch (Exception e) { - if (e instanceof ElasticsearchException) { - throw (ElasticsearchException) e; - } else { - throw new ElasticsearchException(e); - } - } - } - - private static OrdinalMap buildOrdinalMap(AtomicParentChildFieldData[] atomicFD, String parentType) throws IOException { - final SortedDocValues[] ordinals = new SortedDocValues[atomicFD.length]; - for (int i = 0; i < ordinals.length; ++i) { - ordinals[i] = atomicFD[i].getOrdinalsValues(parentType); - } - return OrdinalMap.build(null, ordinals, PackedInts.DEFAULT); - } - - private static class OrdinalMapAndAtomicFieldData { - final OrdinalMap ordMap; - final AtomicParentChildFieldData[] fieldData; - - OrdinalMapAndAtomicFieldData(OrdinalMap ordMap, AtomicParentChildFieldData[] fieldData) { - this.ordMap = ordMap; - this.fieldData = fieldData; - } - } - - @Override - public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { - final long startTime = System.nanoTime(); - - long ramBytesUsed = 0; - final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>(); - for (String type : parentTypes) { - final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()]; - for (LeafReaderContext context : indexReader.leaves()) { - fieldData[context.ord] = load(context); - } - final OrdinalMap ordMap = buildOrdinalMap(fieldData, type); - ramBytesUsed += ordMap.ramBytesUsed(); - perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData)); - } - - final AtomicParentChildFieldData[] fielddata = new AtomicParentChildFieldData[indexReader.leaves().size()]; - for (int i = 0; i < fielddata.length; ++i) { - fielddata[i] = new GlobalAtomicFieldData(parentTypes, perType, i); - } - - breakerService.getBreaker(CircuitBreaker.FIELDDATA).addWithoutBreaking(ramBytesUsed); - if (logger.isDebugEnabled()) { - logger.debug( - "global-ordinals [_parent] took [{}]", - new TimeValue(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) - ); - } - - return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, perType); - } - - private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData { - - private final Set<String> types; - private final Map<String, OrdinalMapAndAtomicFieldData> atomicFD; - private final int segmentIndex; - - GlobalAtomicFieldData(Set<String> types, Map<String, OrdinalMapAndAtomicFieldData> atomicFD, int segmentIndex) { - this.types = types; - this.atomicFD = atomicFD; - this.segmentIndex = segmentIndex; - } - - @Override - public Set<String> types() { - return types; - } - - @Override - public SortedDocValues getOrdinalsValues(String type) { - final OrdinalMapAndAtomicFieldData atomicFD = this.atomicFD.get(type); - if (atomicFD == null) { - return DocValues.emptySorted(); - } - - final OrdinalMap ordMap = atomicFD.ordMap; - final SortedDocValues[] allSegmentValues = new SortedDocValues[atomicFD.fieldData.length]; - for (int i = 0; i < allSegmentValues.length; ++i) { - allSegmentValues[i] = atomicFD.fieldData[i].getOrdinalsValues(type); - } - final SortedDocValues segmentValues = allSegmentValues[segmentIndex]; - if (segmentValues.getValueCount() == ordMap.getValueCount()) { - // ords are already global - return segmentValues; - } - final LongValues globalOrds = ordMap.getGlobalOrds(segmentIndex); - return new AbstractSortedDocValues() { - - @Override - public BytesRef lookupOrd(int ord) throws IOException { - final int segmentIndex = ordMap.getFirstSegmentNumber(ord); - final int segmentOrd = (int) ordMap.getFirstSegmentOrd(ord); - return allSegmentValues[segmentIndex].lookupOrd(segmentOrd); - } - - @Override - public int getValueCount() { - return (int) ordMap.getValueCount(); - } - - @Override - public int ordValue() throws IOException { - return (int) globalOrds.get(segmentValues.ordValue()); - } - - @Override - public boolean advanceExact(int target) throws IOException { - return segmentValues.advanceExact(target); - } - - @Override - public int docID() { - return segmentValues.docID(); - } - }; - } - - @Override - public long ramBytesUsed() { - // this class does not take memory on its own, the index-level field data does - // it through the use of ordinal maps - return 0; - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public void close() { - List<Releasable> closeables = new ArrayList<>(); - for (OrdinalMapAndAtomicFieldData fds : atomicFD.values()) { - closeables.addAll(Arrays.asList(fds.fieldData)); - } - Releasables.close(closeables); - } - - } - - public class GlobalFieldData implements IndexParentChildFieldData, Accountable { - - private final Object coreCacheKey; - private final List<LeafReaderContext> leaves; - private final AtomicParentChildFieldData[] fielddata; - private final long ramBytesUsed; - private final Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType; - - GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMapAndAtomicFieldData> ordinalMapPerType) { - this.coreCacheKey = reader.getReaderCacheHelper().getKey(); - this.leaves = reader.leaves(); - this.ramBytesUsed = ramBytesUsed; - this.fielddata = fielddata; - this.ordinalMapPerType = ordinalMapPerType; - } - - @Override - public String getFieldName() { - return ParentChildIndexFieldData.this.getFieldName(); - } - - @Override - public AtomicParentChildFieldData load(LeafReaderContext context) { - assert context.reader().getCoreCacheHelper().getKey() == leaves.get(context.ord) - .reader().getCoreCacheHelper().getKey(); - return fielddata[context.ord]; - } - - @Override - public AtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception { - return load(context); - } - - @Override - public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) { - throw new UnsupportedOperationException("No sorting on global ords"); - } - - @Override - public void clear() { - ParentChildIndexFieldData.this.clear(); - } - - @Override - public Index index() { - return ParentChildIndexFieldData.this.index(); - } - - @Override - public long ramBytesUsed() { - return ramBytesUsed; - } - - @Override - public Collection<Accountable> getChildResources() { - return Collections.emptyList(); - } - - @Override - public IndexParentChildFieldData loadGlobal(DirectoryReader indexReader) { - if (indexReader.getReaderCacheHelper().getKey() == coreCacheKey) { - return this; - } - throw new IllegalStateException(); - } - - @Override - public IndexParentChildFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { - return loadGlobal(indexReader); - } - - } - - /** - * Returns the global ordinal map for the specified type - */ - // TODO: OrdinalMap isn't expose in the field data framework, because it is an implementation detail. - // However the JoinUtil works directly with OrdinalMap, so this is a hack to get access to OrdinalMap - // I don't think we should expose OrdinalMap in IndexFieldData, because only parent/child relies on it and for the - // rest of the code OrdinalMap is an implementation detail, but maybe we can expose it in IndexParentChildFieldData interface? - public static MultiDocValues.OrdinalMap getOrdinalMap(IndexParentChildFieldData indexParentChildFieldData, String type) { - if (indexParentChildFieldData instanceof ParentChildIndexFieldData.GlobalFieldData) { - return ((GlobalFieldData) indexParentChildFieldData).ordinalMapPerType.get(type).ordMap; - } else { - // one segment, local ordinals are global - return null; - } - } - -} diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java index ea076d476d..9e6e2e994c 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedSetDVOrdinalsIndexFieldData.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedSetSelector; @@ -125,4 +126,9 @@ public class SortedSetDVOrdinalsIndexFieldData extends DocValuesIndexFieldData i public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception { return GlobalOrdinalsBuilder.build(indexReader, this, indexSettings, breakerService, logger, scriptFunction); } + + @Override + public MultiDocValues.OrdinalMap getOrdinalMap() { + return null; + } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java index 62877567c3..3ef9b73708 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ParentFieldMapper.java @@ -37,7 +37,7 @@ import org.elasticsearch.common.settings.loader.SettingsLoader; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; +import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -196,7 +196,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { @Override public IndexFieldData.Builder fielddataBuilder() { - return new ParentChildIndexFieldData.Builder(); + return new DocValuesIndexFieldData.Builder(); } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index e6ed81c2d8..a8aaa25940 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.IndexSearcher; @@ -32,18 +31,15 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.lucene.ScorerAware; import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.AtomicOrdinalsFieldData; -import org.elasticsearch.index.fielddata.AtomicParentChildFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.IndexParentChildFieldData; import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.aggregations.support.ValuesSource.WithScript.BytesValues; @@ -150,40 +146,6 @@ public abstract class ValuesSource { } } - public static class ParentChild extends Bytes { - - protected final ParentChildIndexFieldData indexFieldData; - - public ParentChild(ParentChildIndexFieldData indexFieldData) { - this.indexFieldData = indexFieldData; - } - - public long globalMaxOrd(IndexSearcher indexSearcher, String type) { - DirectoryReader indexReader = (DirectoryReader) indexSearcher.getIndexReader(); - if (indexReader.leaves().isEmpty()) { - return 0; - } else { - LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); - IndexParentChildFieldData globalFieldData = indexFieldData.loadGlobal(indexReader); - AtomicParentChildFieldData afd = globalFieldData.load(atomicReaderContext); - SortedDocValues values = afd.getOrdinalsValues(type); - return values.getValueCount(); - } - } - - public SortedDocValues globalOrdinalsValues(String type, LeafReaderContext context) { - final IndexParentChildFieldData global = indexFieldData.loadGlobal((DirectoryReader)context.parent.reader()); - final AtomicParentChildFieldData atomicFieldData = global.load(context); - return atomicFieldData.getOrdinalsValues(type); - } - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) { - final AtomicParentChildFieldData atomicFieldData = indexFieldData.load(context); - return atomicFieldData.getBytesValues(); - } - } - public static class FieldData extends Bytes { protected final IndexFieldData<?> indexFieldData; diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 39b97a9b02..e5fac62840 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -26,7 +26,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; @@ -317,9 +316,7 @@ public class ValuesSourceConfig<VS extends ValuesSource> { private ValuesSource bytesField() throws IOException { final IndexFieldData<?> indexFieldData = fieldContext().indexFieldData(); ValuesSource dataSource; - if (indexFieldData instanceof ParentChildIndexFieldData) { - dataSource = new ValuesSource.Bytes.WithOrdinals.ParentChild((ParentChildIndexFieldData) indexFieldData); - } else if (indexFieldData instanceof IndexOrdinalsFieldData) { + if (indexFieldData instanceof IndexOrdinalsFieldData) { dataSource = new ValuesSource.Bytes.WithOrdinals.FieldData((IndexOrdinalsFieldData) indexFieldData); } else { dataSource = new ValuesSource.Bytes.FieldData(indexFieldData); diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java index aebb0a802e..05117550ac 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -457,6 +457,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI refreshReader(); IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues()); IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader); + assertNotNull(globalOrdinals.getOrdinalMap()); assertThat(topLevelReader.leaves().size(), equalTo(3)); // First segment @@ -584,6 +585,7 @@ public abstract class AbstractStringFieldDataTestCase extends AbstractFieldDataI refreshReader(); IndexOrdinalsFieldData ifd = getForField("string", "value", hasDocValues()); IndexOrdinalsFieldData globalOrdinals = ifd.loadGlobal(topLevelReader); + assertNotNull(globalOrdinals.getOrdinalMap()); assertThat(ifd.loadGlobal(topLevelReader), sameInstance(globalOrdinals)); // 3 b/c 1 segment level caches and 1 top level cache // in case of doc values, we don't cache atomic FD, so only the top-level cache is there diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java deleted file mode 100644 index f7bdc8efed..0000000000 --- a/core/src/test/java/org/elasticsearch/index/fielddata/ParentChildFieldDataTests.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.SortedDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.ParentFieldMapper; -import org.elasticsearch.index.mapper.Uid; -import org.elasticsearch.index.mapper.UidFieldMapper; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.MultiValueMode; -import org.junit.Before; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicReference; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; - -public class ParentChildFieldDataTests extends AbstractFieldDataTestCase { - private final String parentType = "parent"; - private final String childType = "child"; - private final String grandChildType = "grand-child"; - - @Before - public void setupData() throws Exception { - mapperService.merge( - childType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), MapperService.MergeReason.MAPPING_UPDATE, false - ); - mapperService.merge( - grandChildType, new CompressedXContent(PutMappingRequest.buildFromSimplifiedDef(grandChildType, "_parent", "type=" + childType).string()), MapperService.MergeReason.MAPPING_UPDATE, false - ); - - Document d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "2")); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "3"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "3")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); - d.add(createJoinField(parentType, "2")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "4"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "2"), Field.Store.NO)); - d.add(createJoinField(parentType, "2")); - d.add(createJoinField(childType, "4")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(childType, "5"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(parentType, "1"), Field.Store.NO)); - d.add(createJoinField(parentType, "1")); - d.add(createJoinField(childType, "5")); - writer.addDocument(d); - writer.commit(); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid(grandChildType, "6"), Field.Store.NO)); - d.add(new StringField(ParentFieldMapper.NAME, Uid.createUid(childType, "2"), Field.Store.NO)); - d.add(createJoinField(childType, "2")); - writer.addDocument(d); - - d = new Document(); - d.add(new StringField(UidFieldMapper.NAME, Uid.createUid("other-type", "1"), Field.Store.NO)); - writer.addDocument(d); - } - - private SortedDocValuesField createJoinField(String parentType, String id) { - return new SortedDocValuesField(ParentFieldMapper.joinField(parentType), new BytesRef(id)); - } - - public void testGetBytesValues() throws Exception { - writer.forceMerge(1); // force merge to 1 segment so we can iterate through documents - IndexFieldData indexFieldData = getForField(childType); - List<LeafReaderContext> readerContexts = refreshReader(); - for (LeafReaderContext readerContext : readerContexts) { - AtomicFieldData fieldData = indexFieldData.load(readerContext); - - SortedBinaryDocValues bytesValues = fieldData.getBytesValues(); - assertTrue(bytesValues.advanceExact(0)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - - assertTrue(bytesValues.advanceExact(1)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertTrue(bytesValues.advanceExact(2)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("3")); - - assertTrue(bytesValues.advanceExact(3)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertTrue(bytesValues.advanceExact(4)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("4")); - - assertTrue(bytesValues.advanceExact(5)); - assertThat(bytesValues.docValueCount(), equalTo(2)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("1")); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("5")); - - assertTrue(bytesValues.advanceExact(6)); - assertThat(bytesValues.docValueCount(), equalTo(1)); - assertThat(bytesValues.nextValue().utf8ToString(), equalTo("2")); - - assertFalse(bytesValues.advanceExact(7)); - } - } - - public void testSorting() throws Exception { - IndexFieldData indexFieldData = getForField(parentType); - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(writer)); - SortField sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, false); - TopFieldDocs topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField)); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[1].doc, equalTo(1)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[2].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(5)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[7]).fields[0]), equalTo(null)); - - sortField = indexFieldData.sortField("_last", MultiValueMode.MIN, null, true); - topDocs = searcher.search(new MatchAllDocsQuery(), 10, new Sort(sortField)); - assertThat(topDocs.totalHits, equalTo(8)); - assertThat(topDocs.scoreDocs.length, equalTo(8)); - assertThat(topDocs.scoreDocs[0].doc, equalTo(3)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[0]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[1].doc, equalTo(4)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[1]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[2].doc, equalTo(6)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[2]).fields[0]).utf8ToString(), equalTo("2")); - assertThat(topDocs.scoreDocs[3].doc, equalTo(0)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[3]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[4].doc, equalTo(1)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[4]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[5].doc, equalTo(2)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[5]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[6].doc, equalTo(5)); - assertThat(((BytesRef) ((FieldDoc) topDocs.scoreDocs[6]).fields[0]).utf8ToString(), equalTo("1")); - assertThat(topDocs.scoreDocs[7].doc, equalTo(7)); - assertThat(((FieldDoc) topDocs.scoreDocs[7]).fields[0], nullValue()); - } - - public void testThreads() throws Exception { - final ParentChildIndexFieldData indexFieldData = getForField(childType); - final DirectoryReader reader = ElasticsearchDirectoryReader.wrap( - DirectoryReader.open(writer), new ShardId(new Index("test", ""), 0)); - final IndexParentChildFieldData global = indexFieldData.loadGlobal(reader); - final AtomicReference<Exception> error = new AtomicReference<>(); - final int numThreads = scaledRandomIntBetween(3, 8); - final Thread[] threads = new Thread[numThreads]; - final CountDownLatch latch = new CountDownLatch(1); - - final Map<Object, BytesRef[]> expected = new HashMap<>(); - for (LeafReaderContext context : reader.leaves()) { - AtomicParentChildFieldData leafData = global.load(context); - SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] ids = new BytesRef[parentIds.getValueCount()]; - for (int j = 0; j < parentIds.getValueCount(); ++j) { - final BytesRef id = parentIds.lookupOrd(j); - if (id != null) { - ids[j] = BytesRef.deepCopyOf(id); - } - } - expected.put(context.reader().getCoreCacheHelper().getKey(), ids); - } - - for (int i = 0; i < numThreads; ++i) { - threads[i] = new Thread() { - @Override - public void run() { - try { - latch.await(); - for (int i = 0; i < 100000; ++i) { - for (LeafReaderContext context : reader.leaves()) { - AtomicParentChildFieldData leafData = global.load(context); - SortedDocValues parentIds = leafData.getOrdinalsValues(parentType); - final BytesRef[] expectedIds = expected.get(context.reader().getCoreCacheHelper().getKey()); - for (int j = 0; j < parentIds.getValueCount(); ++j) { - final BytesRef id = parentIds.lookupOrd(j); - assertEquals(expectedIds[j], id); - } - } - } - } catch (Exception e) { - error.compareAndSet(null, e); - } - } - }; - threads[i].start(); - } - latch.countDown(); - for (Thread thread : threads) { - thread.join(); - } - if (error.get() != null) { - throw error.get(); - } - } - - @Override - protected String getFieldDataType() { - return "_parent"; - } -} |