diff options
Diffstat (limited to 'core/src/test/java/org/elasticsearch/versioning')
-rw-r--r-- | core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java | 444 |
1 files changed, 204 insertions, 240 deletions
diff --git a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java index 5296e763cc..93c29e0c92 100644 --- a/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java +++ b/core/src/test/java/org/elasticsearch/versioning/SimpleVersioningIT.java @@ -18,15 +18,6 @@ */ package org.elasticsearch.versioning; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Locale; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicInteger; - import org.apache.lucene.util.TestUtil; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.bulk.BulkResponse; @@ -37,12 +28,15 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.VersionType; -import org.elasticsearch.index.engine.DocumentAlreadyExistsException; import org.elasticsearch.index.engine.FlushNotAllowedEngineException; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.test.ESIntegTestCase; import org.junit.Test; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; @@ -100,7 +94,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { } // deleting with a lower version works. - long v= randomIntBetween(12,14); + long v = randomIntBetween(12, 14); DeleteResponse deleteResponse = client().prepareDelete("test", "type", "1").setVersion(v).setVersionType(VersionType.FORCE).get(); assertThat(deleteResponse.isFound(), equalTo(true)); assertThat(deleteResponse.getVersion(), equalTo(v)); @@ -136,7 +130,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { VersionConflictEngineException.class); // Delete with a higher or equal version deletes all versions up to the given one. - long v= randomIntBetween(14,17); + long v = randomIntBetween(14, 17); DeleteResponse deleteResponse = client().prepareDelete("test", "type", "1").setVersion(v).setVersionType(VersionType.EXTERNAL_GTE).execute().actionGet(); assertThat(deleteResponse.isFound(), equalTo(true)); assertThat(deleteResponse.getVersion(), equalTo(v)); @@ -165,7 +159,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(indexResponse.getVersion(), equalTo(14l)); assertThrows(client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(13).setVersionType(VersionType.EXTERNAL).execute(), - VersionConflictEngineException.class); + VersionConflictEngineException.class); if (randomBoolean()) { refresh(); @@ -176,8 +170,8 @@ public class SimpleVersioningIT extends ESIntegTestCase { // deleting with a lower version fails. assertThrows( - client().prepareDelete("test", "type", "1").setVersion(2).setVersionType(VersionType.EXTERNAL).execute(), - VersionConflictEngineException.class); + client().prepareDelete("test", "type", "1").setVersion(2).setVersionType(VersionType.EXTERNAL).execute(), + VersionConflictEngineException.class); // Delete with a higher version deletes all versions up to the given one. DeleteResponse deleteResponse = client().prepareDelete("test", "type", "1").setVersion(17).setVersionType(VersionType.EXTERNAL).execute().actionGet(); @@ -186,8 +180,8 @@ public class SimpleVersioningIT extends ESIntegTestCase { // Deleting with a lower version keeps on failing after a delete. assertThrows( - client().prepareDelete("test", "type", "1").setVersion(2).setVersionType(VersionType.EXTERNAL).execute(), - VersionConflictEngineException.class); + client().prepareDelete("test", "type", "1").setVersion(2).setVersionType(VersionType.EXTERNAL).execute(), + VersionConflictEngineException.class); // But delete with a higher version is OK. @@ -206,8 +200,8 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThat(deleteResponse.getVersion(), equalTo(20l)); // Make sure that the next delete will be GC. Note we do it on the index settings so it will be cleaned up - HashMap<String,Object> newSettings = new HashMap<>(); - newSettings.put("index.gc_deletes",-1); + HashMap<String, Object> newSettings = new HashMap<>(); + newSettings.put("index.gc_deletes", -1); client().admin().indices().prepareUpdateSettings("test").setSettings(newSettings).execute().actionGet(); Thread.sleep(300); // gc works based on estimated sampled time. Give it a chance... @@ -221,7 +215,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { public void testRequireUnitsOnUpdateSettings() throws Exception { createIndex("test"); ensureGreen(); - HashMap<String,Object> newSettings = new HashMap<>(); + HashMap<String, Object> newSettings = new HashMap<>(); newSettings.put("index.gc_deletes", "42"); try { client().admin().indices().prepareUpdateSettings("test").setSettings(newSettings).execute().actionGet(); @@ -262,22 +256,12 @@ public class SimpleVersioningIT extends ESIntegTestCase { VersionConflictEngineException.class); assertThrows( - client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(1).execute(), - VersionConflictEngineException.class); - - assertThrows( - client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(1).execute(), - VersionConflictEngineException.class); - assertThrows( - client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(1).execute(), + client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(1).execute(), VersionConflictEngineException.class); assertThrows( - client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(2).execute(), - DocumentAlreadyExistsException.class); - assertThrows( - client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(2).execute(), - DocumentAlreadyExistsException.class); + client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").execute(), + VersionConflictEngineException.class); assertThrows(client().prepareDelete("test", "type", "1").setVersion(1).execute(), VersionConflictEngineException.class); @@ -334,10 +318,8 @@ public class SimpleVersioningIT extends ESIntegTestCase { assertThrows(client().prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(1).execute(), VersionConflictEngineException.class); - assertThrows(client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(1).execute(), - VersionConflictEngineException.class); - assertThrows(client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").setVersion(1).execute(), + assertThrows(client().prepareIndex("test", "type", "1").setCreate(true).setSource("field1", "value1_1").execute(), VersionConflictEngineException.class); assertThrows(client().prepareDelete("test", "type", "1").setVersion(1).execute(), VersionConflictEngineException.class); @@ -377,90 +359,94 @@ public class SimpleVersioningIT extends ESIntegTestCase { IDSource ids; final Random random = getRandom(); switch (random.nextInt(6)) { - case 0: - // random simple - if (VERBOSE) { - System.out.println("TEST: use random simple ids"); - } - ids = new IDSource() { + case 0: + // random simple + if (VERBOSE) { + System.out.println("TEST: use random simple ids"); + } + ids = new IDSource() { @Override public String next() { return TestUtil.randomSimpleString(random); } }; - break; - case 1: - // random realistic unicode - if (VERBOSE) { - System.out.println("TEST: use random realistic unicode ids"); - } - ids = new IDSource() { + break; + case 1: + // random realistic unicode + if (VERBOSE) { + System.out.println("TEST: use random realistic unicode ids"); + } + ids = new IDSource() { @Override public String next() { return TestUtil.randomRealisticUnicodeString(random); } }; - break; - case 2: - // sequential - if (VERBOSE) { - System.out.println("TEST: use seuquential ids"); - } - ids = new IDSource() { + break; + case 2: + // sequential + if (VERBOSE) { + System.out.println("TEST: use seuquential ids"); + } + ids = new IDSource() { int upto; + @Override public String next() { return Integer.toString(upto++); } }; - break; - case 3: - // zero-pad sequential - if (VERBOSE) { - System.out.println("TEST: use zero-pad seuquential ids"); - } - ids = new IDSource() { + break; + case 3: + // zero-pad sequential + if (VERBOSE) { + System.out.println("TEST: use zero-pad seuquential ids"); + } + ids = new IDSource() { final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX); final String zeroPad = String.format(Locale.ROOT, "%0" + TestUtil.nextInt(random, 4, 20) + "d", 0); int upto; + @Override public String next() { String s = Integer.toString(upto++); return zeroPad.substring(zeroPad.length() - s.length()) + s; } }; - break; - case 4: - // random long - if (VERBOSE) { - System.out.println("TEST: use random long ids"); - } - ids = new IDSource() { + break; + case 4: + // random long + if (VERBOSE) { + System.out.println("TEST: use random long ids"); + } + ids = new IDSource() { final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX); int upto; + @Override public String next() { return Long.toString(random.nextLong() & 0x3ffffffffffffffL, radix); } }; - break; - case 5: - // zero-pad random long - if (VERBOSE) { - System.out.println("TEST: use zero-pad random long ids"); - } - ids = new IDSource() { + break; + case 5: + // zero-pad random long + if (VERBOSE) { + System.out.println("TEST: use zero-pad random long ids"); + } + ids = new IDSource() { final int radix = TestUtil.nextInt(random, Character.MIN_RADIX, Character.MAX_RADIX); final String zeroPad = String.format(Locale.ROOT, "%015d", 0); int upto; + @Override public String next() { return Long.toString(random.nextLong() & 0x3ffffffffffffffL, radix); } }; - break; - default: - throw new AssertionError(); + break; + default: + throw new AssertionError(); } return ids; @@ -530,7 +516,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { } else { sb.append(" response: null"); } - + return sb.toString(); } } @@ -547,7 +533,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { // TODO: not great we don't test deletes GC here: // We test deletes, but can't rely on wall-clock delete GC: - HashMap<String,Object> newSettings = new HashMap<>(); + HashMap<String, Object> newSettings = new HashMap<>(); newSettings.put("index.gc_deletes", "1000000h"); assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(newSettings).execute().actionGet()); @@ -584,14 +570,14 @@ public class SimpleVersioningIT extends ESIntegTestCase { // Attach random versions to them: long version = 0; - final IDAndVersion[] idVersions = new IDAndVersion[TestUtil.nextInt(random, numIDs/2, numIDs*(TEST_NIGHTLY ? 8 : 2))]; - final Map<String,IDAndVersion> truth = new HashMap<>(); + final IDAndVersion[] idVersions = new IDAndVersion[TestUtil.nextInt(random, numIDs / 2, numIDs * (TEST_NIGHTLY ? 8 : 2))]; + final Map<String, IDAndVersion> truth = new HashMap<>(); if (VERBOSE) { System.out.println("TEST: use " + numIDs + " ids; " + idVersions.length + " operations"); } - for(int i=0;i<idVersions.length;i++) { + for (int i = 0; i < idVersions.length; i++) { if (useMonotonicVersion) { version += TestUtil.nextInt(random, 1, 10); @@ -612,7 +598,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { } // Shuffle - for(int i = idVersions.length - 1; i > 0; i--) { + for (int i = idVersions.length - 1; i > 0; i--) { int index = random.nextInt(i + 1); IDAndVersion x = idVersions[index]; idVersions[index] = idVersions[i]; @@ -620,7 +606,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { } if (VERBOSE) { - for(IDAndVersion idVersion : idVersions) { + for (IDAndVersion idVersion : idVersions) { System.out.println("id=" + idVersion.id + " version=" + idVersion.version + " delete?=" + idVersion.delete + " truth?=" + (truth.get(idVersion.id) == idVersion)); } } @@ -629,109 +615,87 @@ public class SimpleVersioningIT extends ESIntegTestCase { final CountDownLatch startingGun = new CountDownLatch(1); Thread[] threads = new Thread[TestUtil.nextInt(random, 1, TEST_NIGHTLY ? 20 : 5)]; final long startTime = System.nanoTime(); - for(int i=0;i<threads.length;i++) { + for (int i = 0; i < threads.length; i++) { final int threadID = i; threads[i] = new Thread() { - @Override - public void run() { - try { - //final Random threadRandom = RandomizedContext.current().getRandom(); - final Random threadRandom = getRandom(); - startingGun.await(); - while (true) { - - // TODO: sometimes use bulk: - - int index = upto.getAndIncrement(); - if (index >= idVersions.length) { - break; - } - if (VERBOSE && index % 100 == 0) { - System.out.println(Thread.currentThread().getName() + ": index=" + index); - } - IDAndVersion idVersion = idVersions[index]; - - String id = idVersion.id; - idVersion.threadID = threadID; - idVersion.indexStartTime = System.nanoTime()-startTime; - long version = idVersion.version; - if (idVersion.delete) { - try { - idVersion.response = client().prepareDelete("test", "type", id) + @Override + public void run() { + try { + //final Random threadRandom = RandomizedContext.current().getRandom(); + final Random threadRandom = getRandom(); + startingGun.await(); + while (true) { + + // TODO: sometimes use bulk: + + int index = upto.getAndIncrement(); + if (index >= idVersions.length) { + break; + } + if (VERBOSE && index % 100 == 0) { + System.out.println(Thread.currentThread().getName() + ": index=" + index); + } + IDAndVersion idVersion = idVersions[index]; + + String id = idVersion.id; + idVersion.threadID = threadID; + idVersion.indexStartTime = System.nanoTime() - startTime; + long version = idVersion.version; + if (idVersion.delete) { + try { + idVersion.response = client().prepareDelete("test", "type", id) .setVersion(version) .setVersionType(VersionType.EXTERNAL).execute().actionGet(); - } catch (VersionConflictEngineException vcee) { - // OK: our version is too old - assertThat(version, lessThanOrEqualTo(truth.get(id).version)); - idVersion.versionConflict = true; - } - } else { - for (int x=0;x<2;x++) { - // Try create first: - - IndexRequest.OpType op; - if (x == 0) { - op = IndexRequest.OpType.CREATE; - } else { - op = IndexRequest.OpType.INDEX; - } - - // index document - try { - idVersion.response = client().prepareIndex("test", "type", id) - .setSource("foo", "bar") - .setOpType(op) - .setVersion(version) - .setVersionType(VersionType.EXTERNAL).execute().actionGet(); - break; - } catch (DocumentAlreadyExistsException daee) { - if (x == 0) { - // OK: id was already indexed by another thread, now use index: - idVersion.alreadyExists = true; - } else { - // Should not happen with op=INDEX: - throw daee; - } - } catch (VersionConflictEngineException vcee) { - // OK: our version is too old - assertThat(version, lessThanOrEqualTo(truth.get(id).version)); - idVersion.versionConflict = true; - } - } + } catch (VersionConflictEngineException vcee) { + // OK: our version is too old + assertThat(version, lessThanOrEqualTo(truth.get(id).version)); + idVersion.versionConflict = true; } - idVersion.indexFinishTime = System.nanoTime()-startTime; - - if (threadRandom.nextInt(100) == 7) { - System.out.println(threadID + ": TEST: now refresh at " + (System.nanoTime()-startTime)); - refresh(); - System.out.println(threadID + ": TEST: refresh done at " + (System.nanoTime()-startTime)); + } else { + try { + idVersion.response = client().prepareIndex("test", "type", id) + .setSource("foo", "bar") + .setVersion(version).setVersionType(VersionType.EXTERNAL).get(); + + } catch (VersionConflictEngineException vcee) { + // OK: our version is too old + assertThat(version, lessThanOrEqualTo(truth.get(id).version)); + idVersion.versionConflict = true; } - if (threadRandom.nextInt(100) == 7) { - System.out.println(threadID + ": TEST: now flush at " + (System.nanoTime()-startTime)); - try { - flush(); - } catch (FlushNotAllowedEngineException fnaee) { - // OK - } - System.out.println(threadID + ": TEST: flush done at " + (System.nanoTime()-startTime)); + } + idVersion.indexFinishTime = System.nanoTime() - startTime; + + if (threadRandom.nextInt(100) == 7) { + System.out.println(threadID + ": TEST: now refresh at " + (System.nanoTime() - startTime)); + refresh(); + System.out.println(threadID + ": TEST: refresh done at " + (System.nanoTime() - startTime)); + } + if (threadRandom.nextInt(100) == 7) { + System.out.println(threadID + ": TEST: now flush at " + (System.nanoTime() - startTime)); + try { + flush(); + } catch (FlushNotAllowedEngineException fnaee) { + // OK } + System.out.println(threadID + ": TEST: flush done at " + (System.nanoTime() - startTime)); } - } catch (Exception e) { - throw new RuntimeException(e); } + } catch (Exception e) { + throw new RuntimeException(e); } - }; + } + }; threads[i].start(); } startingGun.countDown(); - for(Thread thread : threads) { + for (Thread thread : threads) { thread.join(); } // Verify against truth: boolean failed = false; - for(String id : ids) { + for (String id : ids) { long expected; IDAndVersion idVersion = truth.get(id); if (idVersion != null && idVersion.delete == false) { @@ -748,7 +712,7 @@ public class SimpleVersioningIT extends ESIntegTestCase { if (failed) { System.out.println("All versions:"); - for(int i=0;i<idVersions.length;i++) { + for (int i = 0; i < idVersions.length; i++) { System.out.println("i=" + i + " " + idVersions[i]); } fail("wrong versions for some IDs"); @@ -760,36 +724,36 @@ public class SimpleVersioningIT extends ESIntegTestCase { // We require only one shard for this test, so that the 2nd delete provokes pruning the deletes map: client() - .admin() - .indices() - .prepareCreate("test") - .setSettings(Settings.settingsBuilder() - .put("index.number_of_shards", 1)) - .execute(). - actionGet(); + .admin() + .indices() + .prepareCreate("test") + .setSettings(Settings.settingsBuilder() + .put("index.number_of_shards", 1)) + .execute(). + actionGet(); ensureGreen(); - HashMap<String,Object> newSettings = new HashMap<>(); + HashMap<String, Object> newSettings = new HashMap<>(); newSettings.put("index.gc_deletes", "10ms"); newSettings.put("index.refresh_interval", "-1"); client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(newSettings) - .execute() - .actionGet(); + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(newSettings) + .execute() + .actionGet(); // Index a doc: client() - .prepareIndex("test", "type", "id") - .setSource("foo", "bar") - .setOpType(IndexRequest.OpType.INDEX) - .setVersion(10) - .setVersionType(VersionType.EXTERNAL) - .execute() - .actionGet(); + .prepareIndex("test", "type", "id") + .setSource("foo", "bar") + .setOpType(IndexRequest.OpType.INDEX) + .setVersion(10) + .setVersionType(VersionType.EXTERNAL) + .execute() + .actionGet(); if (randomBoolean()) { // Force refresh so the add is sometimes visible in the searcher: @@ -798,20 +762,20 @@ public class SimpleVersioningIT extends ESIntegTestCase { // Delete it client() - .prepareDelete("test", "type", "id") - .setVersion(11) - .setVersionType(VersionType.EXTERNAL) - .execute() - .actionGet(); + .prepareDelete("test", "type", "id") + .setVersion(11) + .setVersionType(VersionType.EXTERNAL) + .execute() + .actionGet(); // Real-time get should reflect delete: assertThat("doc should have been deleted", - client() - .prepareGet("test", "type", "id") - .execute() - .actionGet() - .getVersion(), - equalTo(-1L)); + client() + .prepareGet("test", "type", "id") + .execute() + .actionGet() + .getVersion(), + equalTo(-1L)); // ThreadPool.estimatedTimeInMillis has default granularity of 200 msec, so we must sleep at least that long; sleep much longer in // case system is busy: @@ -819,20 +783,20 @@ public class SimpleVersioningIT extends ESIntegTestCase { // Delete an unrelated doc (provokes pruning deletes from versionMap) client() - .prepareDelete("test", "type", "id2") - .setVersion(11) - .setVersionType(VersionType.EXTERNAL) - .execute() - .actionGet(); + .prepareDelete("test", "type", "id2") + .setVersion(11) + .setVersionType(VersionType.EXTERNAL) + .execute() + .actionGet(); // Real-time get should still reflect delete: assertThat("doc should have been deleted", - client() - .prepareGet("test", "type", "id") - .execute() - .actionGet() - .getVersion(), - equalTo(-1L)); + client() + .prepareGet("test", "type", "id") + .execute() + .actionGet() + .getVersion(), + equalTo(-1L)); } @Test @@ -842,25 +806,25 @@ public class SimpleVersioningIT extends ESIntegTestCase { ensureGreen(); // We test deletes, but can't rely on wall-clock delete GC: - HashMap<String,Object> newSettings = new HashMap<>(); + HashMap<String, Object> newSettings = new HashMap<>(); newSettings.put("index.gc_deletes", "0ms"); client() - .admin() - .indices() - .prepareUpdateSettings("test") - .setSettings(newSettings) - .execute() - .actionGet(); + .admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(newSettings) + .execute() + .actionGet(); // Index a doc: client() - .prepareIndex("test", "type", "id") - .setSource("foo", "bar") - .setOpType(IndexRequest.OpType.INDEX) - .setVersion(10) - .setVersionType(VersionType.EXTERNAL) - .execute() - .actionGet(); + .prepareIndex("test", "type", "id") + .setSource("foo", "bar") + .setOpType(IndexRequest.OpType.INDEX) + .setVersion(10) + .setVersionType(VersionType.EXTERNAL) + .execute() + .actionGet(); if (randomBoolean()) { // Force refresh so the add is sometimes visible in the searcher: @@ -869,19 +833,19 @@ public class SimpleVersioningIT extends ESIntegTestCase { // Delete it client() - .prepareDelete("test", "type", "id") - .setVersion(11) - .setVersionType(VersionType.EXTERNAL) - .execute() - .actionGet(); + .prepareDelete("test", "type", "id") + .setVersion(11) + .setVersionType(VersionType.EXTERNAL) + .execute() + .actionGet(); // Real-time get should reflect delete even though index.gc_deletes is 0: assertThat("doc should have been deleted", - client() - .prepareGet("test", "type", "id") - .execute() - .actionGet() - .getVersion(), - equalTo(-1L)); + client() + .prepareGet("test", "type", "id") + .execute() + .actionGet() + .getVersion(), + equalTo(-1L)); } } |