From b46f4fb7af06d854668d7754ad1ae4618ae5a803 Mon Sep 17 00:00:00 2001 From: Kristof Beyls Date: Wed, 20 May 2015 07:35:22 +0000 Subject: Disable more short-running programs in benchmark-only mode. A substantial source of performance noisiness of the LNT test-suite are programs running very shortly. The following programs, are disabled in benchmark-only mode by this commit: * SingleSource/UnitTests/Vector: constpool simple: both don't have any loops in the code. * SingleSource/UnitTests/Vector/AArch64: aarch64_neon_intrinsics: doesn't have any loop in the code. * SingleSource/UnitTests/Vector/NEON: simple: doesn't have any loop in the code. * SingleSource/UnitTests: 2005-07-15-Bitfield-ABI 2006-01-23-UnionInit 2007-04-10-BitfieldTest: doesn't have any loop in the code. * MultiSource/Benchmarks/Prolangs-C: loader: This program exits immediately because no arguments are given on the command line. Unless someone creates inputs for this program, this should not be considered a benchmark. * MultiSource/Benchmarks/McCat: - 15-trie produces trie data structures, but the program has no loops so it's probably IO bound, and therefore shouldn't be considered as a benchmark. * MultiSource/Benchmarks/Prolangs-C: - cdecl: The benchmark parses about 70 C declarations * MultiSource/Benchmarks/MiBench: - office-stringsearch searches a few hundred substrings in a set of a few hundred strings. - telecom-adpcm seems to spend most of its time in IO - and should have a loop to do the main data transformation multiple times on the same buffer if run as a benchmark. * SingleSource/Benchmarks/Stanford: - IntMM: does 10 matrix multiplications of size 40x40. * SingleSource/Regression/C/Makefile: - matrixTranspose: transposes a 32x32 matrix 10 times. - sumarray2d: creates a 100x100 matrix and sums all the elements in it. - test_indvars: traverses a 20000-element matrix twice. * SingleSource/UnitTests/SignlessTypes: - rem: does 100 gcd computations. The following programs also run very shortly, but I do think they have value as a benchmark - so they are not disabled by this commit: * SingleSource/Benchmarks/Misc/lowercase * SingleSource/Benchmarks/Shootout/objinst * SingleSource/Benchmarks/Shootout-C++/objinst For all three, it seems that the main computation in the benchmark is completely optimized away. Keeping these running as benchmarks should allow us to catch if llvm ever regresses in being able to optimize away the main computation in these programs. git-svn-id: https://llvm.org/svn/llvm-project/test-suite/trunk@237777 91177308-0d34-0410-b5e6-96231b3b80d8 --- SingleSource/Benchmarks/Stanford/Makefile | 4 ++++ SingleSource/Regression/C/Makefile | 3 +++ SingleSource/UnitTests/Makefile | 3 +++ SingleSource/UnitTests/SignlessTypes/Makefile | 2 +- SingleSource/UnitTests/Vector/AArch64/Makefile | 4 ++++ SingleSource/UnitTests/Vector/Makefile | 2 +- SingleSource/UnitTests/Vector/NEON/Makefile | 4 ++++ 7 files changed, 20 insertions(+), 2 deletions(-) (limited to 'SingleSource') diff --git a/SingleSource/Benchmarks/Stanford/Makefile b/SingleSource/Benchmarks/Stanford/Makefile index 8eebe804..023671c9 100644 --- a/SingleSource/Benchmarks/Stanford/Makefile +++ b/SingleSource/Benchmarks/Stanford/Makefile @@ -2,4 +2,8 @@ LEVEL = ../../.. LDFLAGS += -lm FP_TOLERANCE = 0.001 +ifdef BENCHMARKING_ONLY +PROGRAMS_TO_SKIP += IntMM +endif + include $(LEVEL)/SingleSource/Makefile.singlesrc diff --git a/SingleSource/Regression/C/Makefile b/SingleSource/Regression/C/Makefile index 3ad16349..1cc49737 100644 --- a/SingleSource/Regression/C/Makefile +++ b/SingleSource/Regression/C/Makefile @@ -30,13 +30,16 @@ PROGRAMS_TO_SKIP += \ ConstructorDestructorAttributes \ DuffsDevice \ globalrefs \ + matrixTranspose \ pointer_arithmetic \ PR10189 \ PR1386 \ PR491 \ PR640 \ sumarray \ + sumarray2d \ sumarraymalloc \ + test_indvars \ testtrace endif diff --git a/SingleSource/UnitTests/Makefile b/SingleSource/UnitTests/Makefile index d18c096f..e85d6434 100644 --- a/SingleSource/UnitTests/Makefile +++ b/SingleSource/UnitTests/Makefile @@ -102,8 +102,10 @@ PROGRAMS_TO_SKIP += \ 2005-05-11-Popcount-ffs-fls \ 2005-05-12-Int64ToFP \ 2005-05-13-SDivTwo \ + 2005-07-15-Bitfield-ABI \ 2005-07-17-INT-To-FP \ 2005-11-29-LongSwitch \ + 2006-01-23-UnionInit \ 2006-01-29-SimpleIndirectCall \ 2006-02-04-DivRem \ 2006-12-01-float_varg \ @@ -112,6 +114,7 @@ PROGRAMS_TO_SKIP += \ 2006-12-11-LoadConstants \ 2007-01-04-KNR-Args \ 2007-03-02-VaCopy \ + 2007-04-10-BitfieldTest \ 2007-04-25-weak \ 2008-04-18-LoopBug \ 2008-04-20-LoopBug2 \ diff --git a/SingleSource/UnitTests/SignlessTypes/Makefile b/SingleSource/UnitTests/SignlessTypes/Makefile index a873b41f..bd5b078a 100644 --- a/SingleSource/UnitTests/SignlessTypes/Makefile +++ b/SingleSource/UnitTests/SignlessTypes/Makefile @@ -12,7 +12,7 @@ SEED := $(shell date +%j) RUN_OPTIONS := 31415926 ifdef BENCHMARKING_ONLY -PROGRAMS_TO_SKIP += cast2 cast-bug ccc div factor shr +PROGRAMS_TO_SKIP += cast2 cast-bug ccc div factor shr rem endif LDFLAGS += -lstdc++ diff --git a/SingleSource/UnitTests/Vector/AArch64/Makefile b/SingleSource/UnitTests/Vector/AArch64/Makefile index 037ece24..75b08a33 100644 --- a/SingleSource/UnitTests/Vector/AArch64/Makefile +++ b/SingleSource/UnitTests/Vector/AArch64/Makefile @@ -3,6 +3,10 @@ DIRS = LEVEL = ../../../.. +ifdef BENCHMARKING_ONLY +PROGRAMS_TO_SKIP += aarch64_neon_intrinsics +endif + include $(LEVEL)/SingleSource/Makefile.singlesrc CFLAGS += -std=c99 diff --git a/SingleSource/UnitTests/Vector/Makefile b/SingleSource/UnitTests/Vector/Makefile index 316ce223..c0088535 100644 --- a/SingleSource/UnitTests/Vector/Makefile +++ b/SingleSource/UnitTests/Vector/Makefile @@ -38,7 +38,7 @@ endif endif ifdef BENCHMARKING_ONLY -PROGRAMS_TO_SKIP += build divides sumarray sumarray-dbl +PROGRAMS_TO_SKIP += build constpool divides simple sumarray sumarray-dbl endif include $(LEVEL)/SingleSource/Makefile.singlesrc diff --git a/SingleSource/UnitTests/Vector/NEON/Makefile b/SingleSource/UnitTests/Vector/NEON/Makefile index 910bd893..f1223647 100644 --- a/SingleSource/UnitTests/Vector/NEON/Makefile +++ b/SingleSource/UnitTests/Vector/NEON/Makefile @@ -3,6 +3,10 @@ DIRS = LEVEL = ../../../.. +ifdef BENCHMARKING_ONLY +PROGRAMS_TO_SKIP += simple +endif + include $(LEVEL)/SingleSource/Makefile.singlesrc CFLAGS += -std=c99 -- cgit v1.2.3