From 7c637a0bfef315b11b3f714dc25645d033aa7632 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 5 Jul 2017 15:16:40 +0200 Subject: Ensure `index.mapping.single_type` can only be set on 5.x indices (#25375) In 6.x we prevent multiple types and default to `index.mapping.single_type: false` This change removes the registered setting and ensures that it's preserved for 5.x indices. Relates to #24961 --- .../common/settings/IndexScopedSettings.java | 4 +- .../org/elasticsearch/index/IndexSettings.java | 20 ++++++++- .../elasticsearch/index/mapper/MapperService.java | 12 +---- .../index/mapper/TypeFieldMapper.java | 13 +++--- .../elasticsearch/index/IndexSettingsTests.java | 52 ++++++++++++++++++++-- .../index/engine/InternalEngineTests.java | 2 +- .../rest-api-spec/test/old_cluster/10_basic.yml | 22 +++++++++ .../test/upgraded_cluster/10_basic.yml | 13 ++++++ 8 files changed, 114 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 890a43107c..1b57a2919f 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -138,7 +138,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, - MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, IndexModule.INDEX_STORE_TYPE_SETTING, IndexModule.INDEX_STORE_PRE_LOAD_SETTING, @@ -197,6 +196,9 @@ public final class IndexScopedSettings extends AbstractScopedSettings { case MergePolicyConfig.INDEX_MERGE_ENABLED: case IndexMetaData.INDEX_SHRINK_SOURCE_UUID_KEY: case IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY: + case IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY: + // this was settable in 5.x but not anymore in 6.x so we have to preserve the value ie. make it read-only + // this can be removed in later versions return true; default: return IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getRawKey().match(key); diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index 537344ca65..fc2e476afc 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -169,6 +169,20 @@ public final class IndexSettings { public static final Setting MAX_SLICES_PER_SCROLL = Setting.intSetting("index.max_slices_per_scroll", 1024, 1, Property.Dynamic, Property.IndexScope); + public static final String INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY = "index.mapping.single_type"; + private static final Setting INDEX_MAPPING_SINGLE_TYPE_SETTING; // private - should not be registered + static { + Function defValue = settings -> { + boolean singleType = true; + if (settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null) { + singleType = Version.indexCreated(settings).onOrAfter(Version.V_6_0_0_alpha1); + } + return Boolean.valueOf(singleType).toString(); + }; + INDEX_MAPPING_SINGLE_TYPE_SETTING = Setting.boolSetting(INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, defValue, Property.IndexScope, + Property.Final); + } + private final Index index; private final Version version; private final Logger logger; @@ -300,7 +314,11 @@ public final class IndexSettings { maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL); this.mergePolicyConfig = new MergePolicyConfig(logger, this); this.indexSortConfig = new IndexSortConfig(this); - singleType = scopedSettings.get(MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING); + singleType = INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexMetaData.getSettings()); // get this from metadata - it's not registered + if ((singleType || version.before(Version.V_6_0_0_alpha1)) == false) { + throw new AssertionError(index.toString() + "multiple types are only allowed on pre 6.x indices but version is: [" + + version + "]"); + } scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio); scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING, mergePolicyConfig::setExpungeDeletesAllowed); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 50f424c268..e279e12c40 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -97,17 +97,7 @@ public class MapperService extends AbstractIndexComponent implements Closeable { public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true; public static final Setting INDEX_MAPPER_DYNAMIC_SETTING = Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.Dynamic, Property.IndexScope); - public static final Setting INDEX_MAPPING_SINGLE_TYPE_SETTING; - static { - Function defValue = settings -> { - boolean singleType = true; - if (settings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) != null) { - singleType = Version.indexCreated(settings).onOrAfter(Version.V_6_0_0_alpha1); - } - return Boolean.valueOf(singleType).toString(); - }; - INDEX_MAPPING_SINGLE_TYPE_SETTING = Setting.boolSetting("index.mapping.single_type", defValue, Property.IndexScope, Property.Final); - } + private static ObjectHashSet META_FIELDS = ObjectHashSet.from( "_uid", "_id", "_type", "_all", "_parent", "_routing", "_index", "_size", "_timestamp", "_ttl" diff --git a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java index 2092e2521d..72c94edfd0 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/TypeFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData; import org.elasticsearch.index.fielddata.plain.ConstantIndexFieldData; @@ -85,7 +86,7 @@ public class TypeFieldMapper extends MetadataFieldMapper { @Override public MetadataFieldMapper getDefault(MappedFieldType fieldType, ParserContext context) { - final Settings indexSettings = context.mapperService().getIndexSettings().getSettings(); + final IndexSettings indexSettings = context.mapperService().getIndexSettings(); return new TypeFieldMapper(indexSettings, fieldType); } } @@ -263,18 +264,18 @@ public class TypeFieldMapper extends MetadataFieldMapper { } } - private TypeFieldMapper(Settings indexSettings, MappedFieldType existing) { + private TypeFieldMapper(IndexSettings indexSettings, MappedFieldType existing) { this(existing == null ? defaultFieldType(indexSettings) : existing.clone(), indexSettings); } - private TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) { - super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings); + private TypeFieldMapper(MappedFieldType fieldType, IndexSettings indexSettings) { + super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings.getSettings()); } - private static MappedFieldType defaultFieldType(Settings indexSettings) { + private static MappedFieldType defaultFieldType(IndexSettings indexSettings) { MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone(); - if (MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexSettings)) { + if (indexSettings.isSingleType()) { defaultFieldType.setIndexOptions(IndexOptions.NONE); defaultFieldType.setHasDocValues(false); } else { diff --git a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java index bc3ee4b5f0..ad1c3f4143 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexSettingsTests.java @@ -288,7 +288,7 @@ public class IndexSettingsTests extends ESTestCase { settings = new IndexSettings(metaData, Settings.EMPTY); assertEquals(IndexSettings.MAX_RESULT_WINDOW_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxResultWindow()); } - + public void testMaxAdjacencyMatrixFiltersSetting() { IndexMetaData metaData = newIndexMeta("index", Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) @@ -301,16 +301,16 @@ public class IndexSettingsTests extends ESTestCase { 42).build())); assertEquals(42, settings.getMaxAdjacencyMatrixFilters()); settings.updateIndexMetaData(newIndexMeta("index", Settings.EMPTY)); - assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), + assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxAdjacencyMatrixFilters()); metaData = newIndexMeta("index", Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .build()); settings = new IndexSettings(metaData, Settings.EMPTY); - assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), + assertEquals(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.get(Settings.EMPTY).intValue(), settings.getMaxAdjacencyMatrixFilters()); - } + } public void testGCDeletesSetting() { TimeValue gcDeleteSetting = new TimeValue(Math.abs(randomInt()), TimeUnit.MILLISECONDS); @@ -435,4 +435,48 @@ public class IndexSettingsTests extends ESTestCase { assertEquals("2s", settings.get("index.refresh_interval")); } + public void testSingleTypeSetting() { + { + IndexSettings index = newIndexSettings(newIndexMeta("index", Settings.EMPTY), Settings.EMPTY); + IndexScopedSettings scopedSettings = index.getScopedSettings(); + Settings build = Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build(); + scopedSettings.archiveUnknownOrInvalidSettings(build, e -> fail("unexpected unknown setting " + e), + (e, ex) -> fail("unexpected illegal setting")); + assertTrue(index.isSingleType()); + expectThrows(IllegalArgumentException.class, () -> { + index.getScopedSettings() + .validate(Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build()); + }); + } + { + boolean single_type = randomBoolean(); + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_6_0) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, single_type) + .build(); + IndexMetaData meta = IndexMetaData.builder("index").settings(settings).build(); + IndexSettings index = newIndexSettings(meta, Settings.EMPTY); + IndexScopedSettings scopedSettings = index.getScopedSettings(); + Settings build = Settings.builder().put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, randomBoolean()).build(); + scopedSettings.archiveUnknownOrInvalidSettings(build, e -> fail("unexpected unknown setting " + e), + (e, ex) -> fail("unexpected illegal setting")); + assertEquals(single_type, index.isSingleType()); + } + + { + Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, false) + .build(); + IndexMetaData meta = IndexMetaData.builder("index").settings(settings).build(); + try { + newIndexSettings(meta, Settings.EMPTY); + fail("should fail with assertion error"); + } catch (AssertionError e) { + // all is well + } + } + } } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index e9c8916634..8030c70a97 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -1486,7 +1486,7 @@ public class InternalEngineTests extends ESTestCase { .put(IndexSettings.INDEX_GC_DELETES_SETTING.getKey(), "1h") // make sure this doesn't kick in on us .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_5_4_0) - .put(MapperService.INDEX_MAPPING_SINGLE_TYPE_SETTING.getKey(), true) + .put(IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY, true) .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.getKey(), between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY))) .build()); diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index 14c25aa101..e531216079 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -7,6 +7,28 @@ settings: index: number_of_replicas: 0 + - do: + indices.create: + index: multi_type_index + body: + settings: + index.number_of_replicas: 0 + index.mapping.single_type: false + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "multi_type_index", "_type": "type1"}}' + - '{"f1": "v1_old", "f2": 0}' + - '{"index": {"_index": "multi_type_index", "_type": "type2"}}' + - '{"f1": "v1_old", "f2": 0}' + + - do: + search: + index: multi_type_index + + - match: { hits.total: 2 } - do: indices.create: diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 96a3c4674c..b72b93ac32 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -17,6 +17,19 @@ - match: { hits.total: 5 } # just check we recovered fine + + - do: + search: + index: multi_type_index + + - match: { hits.total: 2 } # just check we recovered fine + + - do: + indices.get_settings: + index: multi_type_index + + - match: { multi_type_index.settings.index.mapping.single_type: "false"} + - do: bulk: refresh: true -- cgit v1.2.3