Michael Hope 2012-12-05 Merge from FSF arm/aarch64-4.7-branch r193937..r194154.

15 files changed, 1265 insertions, 754 deletions

diff --git a/ChangeLog.linaro b/ChangeLog.linaro
index 765a3c91c05..99615957378 100644
--- a/ChangeLog.linaro
+++ b/ChangeLog.linaro
@@ -1,3 +1,100 @@
+2012-12-05  Michael Hope  <michael.hope@linaro.org>
+
+	Merge from FSF arm/aarch64-4.7-branch r193937..r194154.
+
+	Backport arm-aarch64-4.7 r193937:
+	gcc/ChangeLog.aarch64
+	
+	Backport from mainline.
+	2012-11-20  James Greenhalgh  <james.greenhalgh@arm.com>
+	Tejas Belagod  <tejas.belagod@arm.com>
+	
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_simd_builtin_type_bits): Rename to...
+	(aarch64_simd_builtin_type_mode): ...this, make sequential.
+	(aarch64_simd_builtin_datum): Refactor members.
+	(VAR1, VAR2, ..., VAR12): Update accordingly.
+	(aarch64_simd_builtin_data): Include from aarch64-simd-builtins.def.
+	(aarch64_builtins): Update accordingly.
+	(init_aarch64_simd_builtins): Refactor, rename to...
+	(aarch64_init_simd_builtins): ...this.
+	(aarch64_simd_builtin_compare): Remove.
+	(locate_simd_builtin_icode): Likewise.
+	* config/aarch64/aarch64-protos.h (aarch64_init_builtins): New.
+	(aarch64_expand_builtin): Likewise.
+	(aarch64_load_tp): Likewise.
+	* config/aarch64/aarch64-simd-builtins.def: New file.
+	* config/aarch64/aarch64.c (aarch64_init_builtins):
+	Move to aarch64-builtins.c.
+	(aarch64_expand_builtin): Likewise.
+	(aarch64_load_tp): Remove static designation.
+	* config/aarch64/aarch64.h
+	(aarch64_builtins): Move to aarch64-builtins.c.
+
+	Backport arm-aarch64-4.7 r193939:
+	gcc/
+	
+	Backport from mainline.
+	2012-11-26  James Greenhalgh  <james.greenhalgh@arm.com>
+	
+	* config/aarch64/aarch64-builtins.c (aarch64_builtin_decls): New.
+	(aarch64_init_simd_builtins): Store declaration after builtin
+	initialisation.
+	(aarch64_init_builtins): Likewise.
+	(aarch64_builtin_decl): New.
+	* config/aarch64/aarch64-protos.h (aarch64_builtin_decl): New.
+	* config/aarch64/aarch64.c (TARGET_BUILTIN_DECL): Define.
+
+	Backport arm-aarch64-4.7 r194079:
+	[AARCH64-4.7] Refactor constant generation.
+	
+	2012-12-03  Sofiane Naci  <sofiane.naci@arm.com>
+	
+	* config/aarch64/aarch64.c (aarch64_build_constant): Update prototype.
+	Call emit_move_insn	instead of printing movi/movn/movz instructions.
+	Call gen_insv_immdi instead of printing movk instruction.
+	(aarch64_add_constant): Update prototype.
+	Generate RTL instead of printing add/sub instructions.
+	(aarch64_output_mi_thunk): Update calls to aarch64_build_constant
+	and aarch64_add_constant.
+
+	Backport arm-aarch64-4.7 r194089:
+	[AARCH64-4.7] Backport - Add vcond, vcondu support.
+	
+	Backport of revision 192985.
+	
+	gcc/
+	* config/aarch64/aarch64-simd.md
+	(aarch64_simd_bsl<mode>_internal): New pattern.
+	(aarch64_simd_bsl<mode>): Likewise.
+	(aarch64_vcond_internal<mode>): Likewise.
+	(vcondu<mode><mode>): Likewise.
+	(vcond<mode><mode>): Likewise.
+	* config/aarch64/iterators.md (UNSPEC_BSL): Add to define_constants.
+
+	Backport arm-aarch64-4.7 r194131:
+	
+	2012-12-04  Tejas Belagod  <tejas.belagod@arm.com>
+	
+	* config/aarch64/aarch64.c (aarch64_simd_vector_alignment,
+	aarch64_simd_vector_alignment_reachable): New.
+	(TARGET_VECTOR_ALIGNMENT, TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE):
+	Define.
+
+	Backport arm-aarch64-4.7 r194148:
+	AArch64: Fix ICE due to	missing	TYPE_STUB_DECL on builtin va_list.
+	
+	2012-12-04  Marcus Shawcroft  <marcus.shawcroft@arm.com>
+	
+	* config/aarch64/aarch64.c (aarch64_build_builtin_va_list): Set
+	TYPE_STUB_DECL.
+
+	Backport arm-aarch64-4.7 r194153:
+	AArch64-4.7: Backport refactor of sfp-machine.h
+
+	Backport arm-aarch64-4.7 r194154:
+	AArch64-4.7: Backport implement FP_TRAPPING_EXCEPTIONS.
+
 2012-12-05  Yvan Roux  <yvan.roux@linaro.org>
 
 	Merge from FSF GCC 4.7.2 (svn branches/gcc-4_7-branch 194184).
diff --git a/gcc/ChangeLog.aarch64 b/gcc/ChangeLog.aarch64
index b7d675bac50..7ff42f5fc61 100644
--- a/gcc/ChangeLog.aarch64
+++ b/gcc/ChangeLog.aarch64
@@ -1,3 +1,83 @@
+2012-12-04  Marcus Shawcroft  <marcus.shawcroft@arm.com>
+
+	Backport from mainline
+        2012-12-04  Marcus Shawcroft  <marcus.shawcroft@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_build_builtin_va_list): Set
+	TYPE_STUB_DECL.
+
+2012-12-04  Tejas Belagod  <tejas.belagod@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_simd_vector_alignment,
+	aarch64_simd_vector_alignment_reachable): New.
+	(TARGET_VECTOR_ALIGNMENT, TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE):
+	Define.
+
+2012-12-03  James Greenhalgh  <james.greenhalgh@arm.com>
+
+	Backport from mainline.
+	2012-10-30  James Greenhalgh  <james.greenhalgh@arm.com>
+		    Tejas Belagod <tejas.belagod@arm.com>
+
+	* config/aarch64/aarch64-simd.md
+	(aarch64_simd_bsl<mode>_internal): New pattern.
+	(aarch64_simd_bsl<mode>): Likewise.
+	(aarch64_vcond_internal<mode>): Likewise.
+	(vcondu<mode><mode>): Likewise.
+	(vcond<mode><mode>): Likewise.
+	* config/aarch64/iterators.md (UNSPEC_BSL): Add to define_constants.
+
+2012-12-03  Sofiane Naci  <sofiane.naci@arm.com>
+
+	* config/aarch64/aarch64.c (aarch64_build_constant): Update prototype.
+	Call emit_move_insn	instead of printing movi/movn/movz instructions.
+	Call gen_insv_immdi instead of printing movk instruction.
+	(aarch64_add_constant): Update prototype.
+	Generate RTL instead of printing add/sub instructions.
+	(aarch64_output_mi_thunk): Update calls to aarch64_build_constant
+	and aarch64_add_constant.
+
+2012-11-29  James Greenhalgh  <james.greenhalgh@arm.com>
+
+	Backport from mainline.
+	2012-11-26  James Greenhalgh  <james.greenhalgh@arm.com>
+
+	* config/aarch64/aarch64-builtins.c (aarch64_builtin_decls): New.
+	(aarch64_init_simd_builtins): Store declaration after builtin
+	initialisation.
+	(aarch64_init_builtins): Likewise.
+	(aarch64_builtin_decl): New.
+	* config/aarch64/aarch64-protos.h (aarch64_builtin_decl): New.
+	* config/aarch64/aarch64.c (TARGET_BUILTIN_DECL): Define.
+
+2012-11-29  James Greenhalgh  <james.greenhalgh@arm.com>
+
+	Backport from mainline.
+	2012-11-20  James Greenhalgh  <james.greenhalgh@arm.com>
+		    Tejas Belagod  <tejas.belagod@arm.com>
+
+	* config/aarch64/aarch64-builtins.c
+	(aarch64_simd_builtin_type_bits): Rename to...
+	(aarch64_simd_builtin_type_mode): ...this, make sequential.
+	(aarch64_simd_builtin_datum): Refactor members.
+	(VAR1, VAR2, ..., VAR12): Update accordingly.
+	(aarch64_simd_builtin_data): Include from aarch64-simd-builtins.def.
+	(aarch64_builtins): Update accordingly.
+	(init_aarch64_simd_builtins): Refactor, rename to...
+	(aarch64_init_simd_builtins): ...this.
+	(aarch64_simd_builtin_compare): Remove.
+	(locate_simd_builtin_icode): Likewise.
+	* config/aarch64/aarch64-protos.h (aarch64_init_builtins): New.
+	(aarch64_expand_builtin): Likewise.
+	(aarch64_load_tp): Likewise.
+	* config/aarch64/aarch64-simd-builtins.def: New file.
+	* config/aarch64/aarch64.c (aarch64_init_builtins):
+	Move to aarch64-builtins.c.
+	(aarch64_expand_builtin): Likewise.
+	(aarch64_load_tp): Remove static designation.
+	* config/aarch64/aarch64.h
+	(aarch64_builtins): Move to aarch64-builtins.c.
+
 2012-11-22  Marcus Shawcroft  <marcus.shawcroft@arm.com>
 
 	* doc/md.texi (AArch64 family): Remove Utf.
diff --git a/gcc/config/aarch64/aarch64-builtins.c b/gcc/config/aarch64/aarch64-builtins.c
index 429a0dfdbfc..2db7edb235a 100644
--- a/gcc/config/aarch64/aarch64-builtins.c
+++ b/gcc/config/aarch64/aarch64-builtins.c
@@ -31,27 +31,28 @@
 #include "diagnostic-core.h"
 #include "optabs.h"
 
-enum aarch64_simd_builtin_type_bits
+enum aarch64_simd_builtin_type_mode
 {
-  T_V8QI = 0x0001,
-  T_V4HI = 0x0002,
-  T_V2SI = 0x0004,
-  T_V2SF = 0x0008,
-  T_DI = 0x0010,
-  T_DF = 0x0020,
-  T_V16QI = 0x0040,
-  T_V8HI = 0x0080,
-  T_V4SI = 0x0100,
-  T_V4SF = 0x0200,
-  T_V2DI = 0x0400,
-  T_V2DF = 0x0800,
-  T_TI = 0x1000,
-  T_EI = 0x2000,
-  T_OI = 0x4000,
-  T_XI = 0x8000,
-  T_SI = 0x10000,
-  T_HI = 0x20000,
-  T_QI = 0x40000
+  T_V8QI,
+  T_V4HI,
+  T_V2SI,
+  T_V2SF,
+  T_DI,
+  T_DF,
+  T_V16QI,
+  T_V8HI,
+  T_V4SI,
+  T_V4SF,
+  T_V2DI,
+  T_V2DF,
+  T_TI,
+  T_EI,
+  T_OI,
+  T_XI,
+  T_SI,
+  T_HI,
+  T_QI,
+  T_MAX
 };
 
 #define v8qi_UP  T_V8QI
@@ -76,8 +77,6 @@ enum aarch64_simd_builtin_type_bits
 
 #define UP(X) X##_UP
 
-#define T_MAX 19
-
 typedef enum
 {
   AARCH64_SIMD_BINOP,
@@ -124,253 +123,175 @@ typedef struct
 {
   const char *name;
   const aarch64_simd_itype itype;
-  const int bits;
-  const enum insn_code codes[T_MAX];
-  const unsigned int num_vars;
-  unsigned int base_fcode;
+  enum aarch64_simd_builtin_type_mode mode;
+  const enum insn_code code;
+  unsigned int fcode;
 } aarch64_simd_builtin_datum;
 
 #define CF(N, X) CODE_FOR_aarch64_##N##X
 
 #define VAR1(T, N, A) \
-  #N, AARCH64_SIMD_##T, UP (A), { CF (N, A) }, 1, 0
+  {#N, AARCH64_SIMD_##T, UP (A), CF (N, A), 0},
 #define VAR2(T, N, A, B) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B), { CF (N, A), CF (N, B) }, 2, 0
+  VAR1 (T, N, A) \
+  VAR1 (T, N, B)
 #define VAR3(T, N, A, B, C) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C), \
-  { CF (N, A), CF (N, B), CF (N, C) }, 3, 0
+  VAR2 (T, N, A, B) \
+  VAR1 (T, N, C)
 #define VAR4(T, N, A, B, C, D) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D) }, 4, 0
+  VAR3 (T, N, A, B, C) \
+  VAR1 (T, N, D)
 #define VAR5(T, N, A, B, C, D, E) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) | UP (E), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E) }, 5, 0
+  VAR4 (T, N, A, B, C, D) \
+  VAR1 (T, N, E)
 #define VAR6(T, N, A, B, C, D, E, F) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) | UP (E) | UP (F), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F) }, 6, 0
+  VAR5 (T, N, A, B, C, D, E) \
+  VAR1 (T, N, F)
 #define VAR7(T, N, A, B, C, D, E, F, G) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-			| UP (E) | UP (F) | UP (G), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G) }, 7, 0
+  VAR6 (T, N, A, B, C, D, E, F) \
+  VAR1 (T, N, G)
 #define VAR8(T, N, A, B, C, D, E, F, G, H) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-		| UP (E) | UP (F) | UP (G) \
-		| UP (H), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G), CF (N, H) }, 8, 0
+  VAR7 (T, N, A, B, C, D, E, F, G) \
+  VAR1 (T, N, H)
 #define VAR9(T, N, A, B, C, D, E, F, G, H, I) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-		| UP (E) | UP (F) | UP (G) \
-		| UP (H) | UP (I), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G), CF (N, H), CF (N, I) }, 9, 0
+  VAR8 (T, N, A, B, C, D, E, F, G, H) \
+  VAR1 (T, N, I)
 #define VAR10(T, N, A, B, C, D, E, F, G, H, I, J) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-		| UP (E) | UP (F) | UP (G) \
-		| UP (H) | UP (I) | UP (J), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G), CF (N, H), CF (N, I), CF (N, J) }, 10, 0
-
+  VAR9 (T, N, A, B, C, D, E, F, G, H, I) \
+  VAR1 (T, N, J)
 #define VAR11(T, N, A, B, C, D, E, F, G, H, I, J, K) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-		| UP (E) | UP (F) | UP (G) \
-		| UP (H) | UP (I) | UP (J) | UP (K), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G), CF (N, H), CF (N, I), CF (N, J), CF (N, K) }, 11, 0
-
+  VAR10 (T, N, A, B, C, D, E, F, G, H, I, J) \
+  VAR1 (T, N, K)
 #define VAR12(T, N, A, B, C, D, E, F, G, H, I, J, K, L) \
-  #N, AARCH64_SIMD_##T, UP (A) | UP (B) | UP (C) | UP (D) \
-		| UP (E) | UP (F) | UP (G) \
-		| UP (H) | UP (I) | UP (J) | UP (K) | UP (L), \
-  { CF (N, A), CF (N, B), CF (N, C), CF (N, D), CF (N, E), CF (N, F), \
-    CF (N, G), CF (N, H), CF (N, I), CF (N, J), CF (N, K), CF (N, L) }, 12, 0
-
-
-/* The mode entries in the following table correspond to the "key" type of the
-   instruction variant, i.e. equivalent to that which would be specified after
-   the assembler mnemonic, which usually refers to the last vector operand.
-   (Signed/unsigned/polynomial types are not differentiated between though, and
-   are all mapped onto the same mode for a given element size.) The modes
-   listed per instruction should be the same as those defined for that
-   instruction's pattern in aarch64_simd.md.
-   WARNING: Variants should be listed in the same increasing order as
-   aarch64_simd_builtin_type_bits.  */
+  VAR11 (T, N, A, B, C, D, E, F, G, H, I, J, K) \
+  VAR1 (T, N, L)
+
+/* BUILTIN_<ITERATOR> macros should expand to cover the same range of
+   modes as is given for each define_mode_iterator in
+   config/aarch64/iterators.md.  */
+
+#define BUILTIN_DX(T, N) \
+  VAR2 (T, N, di, df)
+#define BUILTIN_SDQ_I(T, N) \
+  VAR4 (T, N, qi, hi, si, di)
+#define BUILTIN_SD_HSI(T, N) \
+  VAR2 (T, N, hi, si)
+#define BUILTIN_V2F(T, N) \
+  VAR2 (T, N, v2sf, v2df)
+#define BUILTIN_VALL(T, N) \
+  VAR10 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, v2sf, v4sf, v2df)
+#define BUILTIN_VB(T, N) \
+  VAR2 (T, N, v8qi, v16qi)
+#define BUILTIN_VD(T, N) \
+  VAR4 (T, N, v8qi, v4hi, v2si, v2sf)
+#define BUILTIN_VDC(T, N) \
+  VAR6 (T, N, v8qi, v4hi, v2si, v2sf, di, df)
+#define BUILTIN_VDIC(T, N) \
+  VAR3 (T, N, v8qi, v4hi, v2si)
+#define BUILTIN_VDN(T, N) \
+  VAR3 (T, N, v4hi, v2si, di)
+#define BUILTIN_VDQ(T, N) \
+  VAR7 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di)
+#define BUILTIN_VDQF(T, N) \
+  VAR3 (T, N, v2sf, v4sf, v2df)
+#define BUILTIN_VDQHS(T, N) \
+  VAR4 (T, N, v4hi, v8hi, v2si, v4si)
+#define BUILTIN_VDQIF(T, N) \
+  VAR9 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2sf, v4sf, v2df)
+#define BUILTIN_VDQM(T, N) \
+  VAR6 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
+#define BUILTIN_VDQV(T, N) \
+  VAR5 (T, N, v8qi, v16qi, v4hi, v8hi, v4si)
+#define BUILTIN_VDQ_BHSI(T, N) \
+  VAR6 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
+#define BUILTIN_VDQ_I(T, N) \
+  VAR7 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di)
+#define BUILTIN_VDW(T, N) \
+  VAR3 (T, N, v8qi, v4hi, v2si)
+#define BUILTIN_VD_BHSI(T, N) \
+  VAR3 (T, N, v8qi, v4hi, v2si)
+#define BUILTIN_VD_HSI(T, N) \
+  VAR2 (T, N, v4hi, v2si)
+#define BUILTIN_VD_RE(T, N) \
+  VAR6 (T, N, v8qi, v4hi, v2si, v2sf, di, df)
+#define BUILTIN_VQ(T, N) \
+  VAR6 (T, N, v16qi, v8hi, v4si, v2di, v4sf, v2df)
+#define BUILTIN_VQN(T, N) \
+  VAR3 (T, N, v8hi, v4si, v2di)
+#define BUILTIN_VQW(T, N) \
+  VAR3 (T, N, v16qi, v8hi, v4si)
+#define BUILTIN_VQ_HSI(T, N) \
+  VAR2 (T, N, v8hi, v4si)
+#define BUILTIN_VQ_S(T, N) \
+  VAR6 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si)
+#define BUILTIN_VSDQ_HSI(T, N) \
+  VAR6 (T, N, v4hi, v8hi, v2si, v4si, hi, si)
+#define BUILTIN_VSDQ_I(T, N) \
+  VAR11 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, qi, hi, si, di)
+#define BUILTIN_VSDQ_I_BHSI(T, N) \
+  VAR10 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, qi, hi, si)
+#define BUILTIN_VSDQ_I_DI(T, N) \
+  VAR8 (T, N, v8qi, v16qi, v4hi, v8hi, v2si, v4si, v2di, di)
+#define BUILTIN_VSD_HSI(T, N) \
+  VAR4 (T, N, v4hi, v2si, hi, si)
+#define BUILTIN_VSQN_HSDI(T, N) \
+  VAR6 (T, N, v8hi, v4si, v2di, hi, si, di)
+#define BUILTIN_VSTRUCT(T, N) \
+  VAR3 (T, N, oi, ci, xi)
 
 static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
-  {VAR6 (CREATE, create, v8qi, v4hi, v2si, v2sf, di, df)},
-  {VAR6 (GETLANE, get_lane_signed,
-	  v8qi, v4hi, v2si, v16qi, v8hi, v4si)},
-  {VAR7 (GETLANE, get_lane_unsigned,
-	  v8qi, v4hi, v2si, v16qi, v8hi, v4si, v2di)},
-  {VAR4 (GETLANE, get_lane, v2sf, di, v4sf, v2df)},
-  {VAR6 (GETLANE, get_dregoi, v8qi, v4hi, v2si, v2sf, di, df)},
-  {VAR6 (GETLANE, get_qregoi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (GETLANE, get_dregci, v8qi, v4hi, v2si, v2sf, di, df)},
-  {VAR6 (GETLANE, get_qregci, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (GETLANE, get_dregxi, v8qi, v4hi, v2si, v2sf, di, df)},
-  {VAR6 (GETLANE, get_qregxi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (SETLANE, set_qregoi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (SETLANE, set_qregci, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (SETLANE, set_qregxi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-
-  {VAR5 (REINTERP, reinterpretv8qi, v8qi, v4hi, v2si, v2sf, di)},
-  {VAR5 (REINTERP, reinterpretv4hi, v8qi, v4hi, v2si, v2sf, di)},
-  {VAR5 (REINTERP, reinterpretv2si, v8qi, v4hi, v2si, v2sf, di)},
-  {VAR5 (REINTERP, reinterpretv2sf, v8qi, v4hi, v2si, v2sf, di)},
-  {VAR5 (REINTERP, reinterpretdi, v8qi, v4hi, v2si, v2sf, di)},
-  {VAR6 (REINTERP, reinterpretv16qi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (REINTERP, reinterpretv8hi, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (REINTERP, reinterpretv4si, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (REINTERP, reinterpretv4sf, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (REINTERP, reinterpretv2di, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR6 (COMBINE, combine, v8qi, v4hi, v2si, v2sf, di, df)},
-
-  {VAR3 (BINOP, saddl, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, uaddl, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, saddl2, v16qi, v8hi, v4si)},
-  {VAR3 (BINOP, uaddl2, v16qi, v8hi, v4si)},
-  {VAR3 (BINOP, saddw, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, uaddw, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, saddw2, v16qi, v8hi, v4si)},
-  {VAR3 (BINOP, uaddw2, v16qi, v8hi, v4si)},
-  {VAR6 (BINOP, shadd, v8qi, v4hi, v2si, v16qi, v8hi, v4si)},
-  {VAR6 (BINOP, uhadd, v8qi, v4hi, v2si, v16qi, v8hi, v4si)},
-  {VAR6 (BINOP, srhadd, v8qi, v4hi, v2si, v16qi, v8hi, v4si)},
-  {VAR6 (BINOP, urhadd, v8qi, v4hi, v2si, v16qi, v8hi, v4si)},
-  {VAR3 (BINOP, addhn, v8hi, v4si, v2di)},
-  {VAR3 (BINOP, raddhn, v8hi, v4si, v2di)},
-  {VAR3 (TERNOP, addhn2, v8hi, v4si, v2di)},
-  {VAR3 (TERNOP, raddhn2, v8hi, v4si, v2di)},
-  {VAR3 (BINOP, ssubl, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, usubl, v8qi, v4hi, v2si)},
-  {VAR3 (BINOP, ssubl2, v16qi, v8hi, v4si) },
-  {VAR3 (BINOP, usubl2, v16qi, v8hi, v4si) },
-  {VAR3 (BINOP, ssubw, v8qi, v4hi, v2si) },
-  {VAR3 (BINOP, usubw, v8qi, v4hi, v2si) },
-  {VAR3 (BINOP, ssubw2, v16qi, v8hi, v4si) },
-  {VAR3 (BINOP, usubw2, v16qi, v8hi, v4si) },
-  {VAR11 (BINOP, sqadd, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR11 (BINOP, uqadd, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR11 (BINOP, sqsub, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR11 (BINOP, uqsub, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR11 (BINOP, suqadd, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR11 (BINOP, usqadd, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi)},
-  {VAR6 (UNOP, sqmovun, di, v8hi, v4si, v2di, si, hi)},
-  {VAR6 (UNOP, sqmovn, di, v8hi, v4si, v2di, si, hi)},
-  {VAR6 (UNOP, uqmovn, di, v8hi, v4si, v2di, si, hi)},
-  {VAR10 (UNOP, sqabs, v8qi, v4hi, v2si, v16qi, v8hi, v4si, v2di, si, hi, qi)},
-  {VAR10 (UNOP, sqneg, v8qi, v4hi, v2si, v16qi, v8hi, v4si, v2di, si, hi, qi)},
-  {VAR2 (BINOP, pmul, v8qi, v16qi)},
-  {VAR4 (TERNOP, sqdmlal, v4hi, v2si, si, hi)},
-  {VAR4 (QUADOP, sqdmlal_lane, v4hi, v2si, si, hi) },
-  {VAR2 (QUADOP, sqdmlal_laneq, v4hi, v2si) },
-  {VAR2 (TERNOP, sqdmlal_n, v4hi, v2si) },
-  {VAR2 (TERNOP, sqdmlal2, v8hi, v4si)},
-  {VAR2 (QUADOP, sqdmlal2_lane, v8hi, v4si) },
-  {VAR2 (QUADOP, sqdmlal2_laneq, v8hi, v4si) },
-  {VAR2 (TERNOP, sqdmlal2_n, v8hi, v4si) },
-  {VAR4 (TERNOP, sqdmlsl, v4hi, v2si, si, hi)},
-  {VAR4 (QUADOP, sqdmlsl_lane, v4hi, v2si, si, hi) },
-  {VAR2 (QUADOP, sqdmlsl_laneq, v4hi, v2si) },
-  {VAR2 (TERNOP, sqdmlsl_n, v4hi, v2si) },
-  {VAR2 (TERNOP, sqdmlsl2, v8hi, v4si)},
-  {VAR2 (QUADOP, sqdmlsl2_lane, v8hi, v4si) },
-  {VAR2 (QUADOP, sqdmlsl2_laneq, v8hi, v4si) },
-  {VAR2 (TERNOP, sqdmlsl2_n, v8hi, v4si) },
-  {VAR4 (BINOP, sqdmull, v4hi, v2si, si, hi)},
-  {VAR4 (TERNOP, sqdmull_lane, v4hi, v2si, si, hi) },
-  {VAR2 (TERNOP, sqdmull_laneq, v4hi, v2si) },
-  {VAR2 (BINOP, sqdmull_n, v4hi, v2si) },
-  {VAR2 (BINOP, sqdmull2, v8hi, v4si) },
-  {VAR2 (TERNOP, sqdmull2_lane, v8hi, v4si) },
-  {VAR2 (TERNOP, sqdmull2_laneq, v8hi, v4si) },
-  {VAR2 (BINOP, sqdmull2_n, v8hi, v4si) },
-  {VAR6 (BINOP, sqdmulh, v4hi, v2si, v8hi, v4si, si, hi)},
-  {VAR6 (BINOP, sqrdmulh, v4hi, v2si, v8hi, v4si, si, hi)},
-  {VAR8 (BINOP, sshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR3 (SHIFTIMM, sshll_n, v8qi, v4hi, v2si) },
-  {VAR3 (SHIFTIMM, ushll_n, v8qi, v4hi, v2si) },
-  {VAR3 (SHIFTIMM, sshll2_n, v16qi, v8hi, v4si) },
-  {VAR3 (SHIFTIMM, ushll2_n, v16qi, v8hi, v4si) },
-  {VAR8 (BINOP, ushl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (BINOP, sshl_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (BINOP, ushl_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR11 (BINOP, sqshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR11 (BINOP, uqshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR8 (BINOP, srshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (BINOP, urshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR11 (BINOP, sqrshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR11 (BINOP, uqrshl, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR8 (SHIFTIMM, sshr_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTIMM, ushr_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTIMM, srshr_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTIMM, urshr_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTACC, ssra_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTACC, usra_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTACC, srsra_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTACC, ursra_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTINSERT, ssri_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTINSERT, usri_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTINSERT, ssli_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR8 (SHIFTINSERT, usli_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  {VAR11 (SHIFTIMM, sqshlu_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR11 (SHIFTIMM, sqshl_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  {VAR11 (SHIFTIMM, uqshl_n, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  { VAR6 (SHIFTIMM, sqshrun_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR6 (SHIFTIMM, sqrshrun_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR6 (SHIFTIMM, sqshrn_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR6 (SHIFTIMM, uqshrn_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR6 (SHIFTIMM, sqrshrn_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR6 (SHIFTIMM, uqrshrn_n, di, v8hi, v4si, v2di, si, hi) },
-  { VAR8 (BINOP, cmeq, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmge, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmgt, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmle, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmlt, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmhs, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmhi, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR8 (BINOP, cmtst, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di) },
-  { VAR6 (TERNOP, sqdmulh_lane, v4hi, v2si, v8hi, v4si, si, hi) },
-  { VAR6 (TERNOP, sqrdmulh_lane, v4hi, v2si, v8hi, v4si, si, hi) },
-  { VAR3 (BINOP, addp, v8qi, v4hi, v2si) },
-  { VAR1 (UNOP, addp, di) },
-  { VAR11 (BINOP, dup_lane, v8qi, v4hi, v2si, di, v16qi, v8hi, v4si, v2di,
-	  si, hi, qi) },
-  { VAR3 (BINOP, fmax, v2sf, v4sf, v2df) },
-  { VAR3 (BINOP, fmin, v2sf, v4sf, v2df) },
-  { VAR6 (BINOP, smax, v8qi, v4hi, v2si, v16qi, v8hi, v4si) },
-  { VAR6 (BINOP, smin, v8qi, v4hi, v2si, v16qi, v8hi, v4si) },
-  { VAR6 (BINOP, umax, v8qi, v4hi, v2si, v16qi, v8hi, v4si) },
-  { VAR6 (BINOP, umin, v8qi, v4hi, v2si, v16qi, v8hi, v4si) },
-  { VAR3 (UNOP, sqrt, v2sf, v4sf, v2df) },
-  {VAR12 (LOADSTRUCT, ld2,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR12 (LOADSTRUCT, ld3,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR12 (LOADSTRUCT, ld4,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR12 (STORESTRUCT, st2,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR12 (STORESTRUCT, st3,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
-  {VAR12 (STORESTRUCT, st4,
-	 v8qi, v4hi, v2si, v2sf, di, df, v16qi, v8hi, v4si, v4sf, v2di, v2df)},
+#include "aarch64-simd-builtins.def"
+};
+
+#undef VAR1
+#define VAR1(T, N, A) \
+  AARCH64_SIMD_BUILTIN_##N##A,
+
+enum aarch64_builtins
+{
+  AARCH64_BUILTIN_MIN,
+  AARCH64_BUILTIN_THREAD_POINTER,
+  AARCH64_SIMD_BUILTIN_BASE,
+#include "aarch64-simd-builtins.def"
+  AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_BUILTIN_BASE
+			      + ARRAY_SIZE (aarch64_simd_builtin_data),
+  AARCH64_BUILTIN_MAX
 };
 
+#undef BUILTIN_DX
+#undef BUILTIN_SDQ_I
+#undef BUILTIN_SD_HSI
+#undef BUILTIN_V2F
+#undef BUILTIN_VALL
+#undef BUILTIN_VB
+#undef BUILTIN_VD
+#undef BUILTIN_VDC
+#undef BUILTIN_VDIC
+#undef BUILTIN_VDN
+#undef BUILTIN_VDQ
+#undef BUILTIN_VDQF
+#undef BUILTIN_VDQHS
+#undef BUILTIN_VDQIF
+#undef BUILTIN_VDQM
+#undef BUILTIN_VDQV
+#undef BUILTIN_VDQ_BHSI
+#undef BUILTIN_VDQ_I
+#undef BUILTIN_VDW
+#undef BUILTIN_VD_BHSI
+#undef BUILTIN_VD_HSI
+#undef BUILTIN_VD_RE
+#undef BUILTIN_VQ
+#undef BUILTIN_VQN
+#undef BUILTIN_VQW
+#undef BUILTIN_VQ_HSI
+#undef BUILTIN_VQ_S
+#undef BUILTIN_VSDQ_HSI
+#undef BUILTIN_VSDQ_I
+#undef BUILTIN_VSDQ_I_BHSI
+#undef BUILTIN_VSDQ_I_DI
+#undef BUILTIN_VSD_HSI
+#undef BUILTIN_VSQN_HSDI
+#undef BUILTIN_VSTRUCT
 #undef CF
 #undef VAR1
 #undef VAR2
@@ -384,13 +305,15 @@ static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
 #undef VAR10
 #undef VAR11
 
+static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
+
 #define NUM_DREG_TYPES 6
 #define NUM_QREG_TYPES 6
 
 void
-init_aarch64_simd_builtins (void)
+aarch64_init_simd_builtins (void)
 {
-  unsigned int i, fcode = AARCH64_SIMD_BUILTIN_BASE;
+  unsigned int i, fcode = AARCH64_SIMD_BUILTIN_BASE + 1;
 
   /* Scalar type nodes.  */
   tree aarch64_simd_intQI_type_node;
@@ -680,417 +603,388 @@ init_aarch64_simd_builtins (void)
 	}
     }
 
-  for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++)
+  for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
     {
       aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
-      unsigned int j, codeidx = 0;
+      const char *const modenames[] =
+      {
+	"v8qi", "v4hi", "v2si", "v2sf", "di", "df",
+	"v16qi", "v8hi", "v4si", "v4sf", "v2di", "v2df",
+	"ti", "ei", "oi", "xi", "si", "hi", "qi"
+      };
+      char namebuf[60];
+      tree ftype = NULL;
+      tree fndecl = NULL;
+      int is_load = 0;
+      int is_store = 0;
+
+      gcc_assert (ARRAY_SIZE (modenames) == T_MAX);
 
-      d->base_fcode = fcode;
+      d->fcode = fcode;
 
-      for (j = 0; j < T_MAX; j++)
+      switch (d->itype)
 	{
-	  const char *const modenames[] = {
-	    "v8qi", "v4hi", "v2si", "v2sf", "di", "df",
-	    "v16qi", "v8hi", "v4si", "v4sf", "v2di", "v2df",
-	    "ti", "ei", "oi", "xi", "si", "hi", "qi"
-	  };
-	  char namebuf[60];
-	  tree ftype = NULL;
-	  enum insn_code icode;
-	  int is_load = 0;
-	  int is_store = 0;
-
-	  /* Skip if particular mode not supported.  */
-	  if ((d->bits & (1 << j)) == 0)
-	    continue;
-
-	  icode = d->codes[codeidx++];
-
-	  switch (d->itype)
-	    {
-	    case AARCH64_SIMD_LOAD1:
-	    case AARCH64_SIMD_LOAD1LANE:
-	    case AARCH64_SIMD_LOADSTRUCTLANE:
-	    case AARCH64_SIMD_LOADSTRUCT:
-	      is_load = 1;
-	      /* Fall through.  */
-	    case AARCH64_SIMD_STORE1:
-	    case AARCH64_SIMD_STORE1LANE:
-	    case AARCH64_SIMD_STORESTRUCTLANE:
-	    case AARCH64_SIMD_STORESTRUCT:
-	      if (!is_load)
-		is_store = 1;
-	      /* Fall through.  */
-	    case AARCH64_SIMD_UNOP:
-	    case AARCH64_SIMD_BINOP:
-	    case AARCH64_SIMD_LOGICBINOP:
-	    case AARCH64_SIMD_SHIFTINSERT:
-	    case AARCH64_SIMD_TERNOP:
-	    case AARCH64_SIMD_QUADOP:
-	    case AARCH64_SIMD_GETLANE:
-	    case AARCH64_SIMD_SETLANE:
-	    case AARCH64_SIMD_CREATE:
-	    case AARCH64_SIMD_DUP:
-	    case AARCH64_SIMD_DUPLANE:
-	    case AARCH64_SIMD_SHIFTIMM:
-	    case AARCH64_SIMD_SHIFTACC:
-	    case AARCH64_SIMD_COMBINE:
-	    case AARCH64_SIMD_SPLIT:
-	    case AARCH64_SIMD_CONVERT:
-	    case AARCH64_SIMD_FIXCONV:
-	    case AARCH64_SIMD_LANEMUL:
-	    case AARCH64_SIMD_LANEMULL:
-	    case AARCH64_SIMD_LANEMULH:
-	    case AARCH64_SIMD_LANEMAC:
-	    case AARCH64_SIMD_SCALARMUL:
-	    case AARCH64_SIMD_SCALARMULL:
-	    case AARCH64_SIMD_SCALARMULH:
-	    case AARCH64_SIMD_SCALARMAC:
-	    case AARCH64_SIMD_SELECT:
-	    case AARCH64_SIMD_VTBL:
-	    case AARCH64_SIMD_VTBX:
+	case AARCH64_SIMD_LOAD1:
+	case AARCH64_SIMD_LOAD1LANE:
+	case AARCH64_SIMD_LOADSTRUCT:
+	case AARCH64_SIMD_LOADSTRUCTLANE:
+	    is_load = 1;
+	  /* Fall through.  */
+	case AARCH64_SIMD_STORE1:
+	case AARCH64_SIMD_STORE1LANE:
+	case AARCH64_SIMD_STORESTRUCT:
+	case AARCH64_SIMD_STORESTRUCTLANE:
+	    if (!is_load)
+	      is_store = 1;
+	  /* Fall through.  */
+	case AARCH64_SIMD_UNOP:
+	case AARCH64_SIMD_BINOP:
+	case AARCH64_SIMD_TERNOP:
+	case AARCH64_SIMD_QUADOP:
+	case AARCH64_SIMD_COMBINE:
+	case AARCH64_SIMD_CONVERT:
+	case AARCH64_SIMD_CREATE:
+	case AARCH64_SIMD_DUP:
+	case AARCH64_SIMD_DUPLANE:
+	case AARCH64_SIMD_FIXCONV:
+	case AARCH64_SIMD_GETLANE:
+	case AARCH64_SIMD_LANEMAC:
+	case AARCH64_SIMD_LANEMUL:
+	case AARCH64_SIMD_LANEMULH:
+	case AARCH64_SIMD_LANEMULL:
+	case AARCH64_SIMD_LOGICBINOP:
+	case AARCH64_SIMD_SCALARMAC:
+	case AARCH64_SIMD_SCALARMUL:
+	case AARCH64_SIMD_SCALARMULH:
+	case AARCH64_SIMD_SCALARMULL:
+	case AARCH64_SIMD_SELECT:
+	case AARCH64_SIMD_SETLANE:
+	case AARCH64_SIMD_SHIFTACC:
+	case AARCH64_SIMD_SHIFTIMM:
+	case AARCH64_SIMD_SHIFTINSERT:
+	case AARCH64_SIMD_SPLIT:
+	case AARCH64_SIMD_VTBL:
+	case AARCH64_SIMD_VTBX:
+	  {
+	    int k;
+	    tree return_type = void_type_node, args = void_list_node;
+	    tree eltype;
+	    /* Build a function type directly from the insn_data for this
+	       builtin.  The build_function_type () function takes care of
+	       removing duplicates for us.  */
+
+	    for (k = insn_data[d->code].n_operands -1; k >= 0; k--)
 	      {
-		int k;
-		tree return_type = void_type_node, args = void_list_node;
+		/* Skip an internal operand for vget_{low, high}.  */
+		if (k == 2 && d->itype == AARCH64_SIMD_SPLIT)
+		  continue;
 
-		/* Build a function type directly from the insn_data for this
-		   builtin.  The build_function_type() function takes care of
-		   removing duplicates for us.  */
-		for (k = insn_data[icode].n_operands - 1; k >= 0; k--)
+		if (is_load && k == 1)
 		  {
-		    tree eltype;
-
-		    /* Skip an internal operand for vget_{low, high}.  */
-		    if (k == 2 && d->itype == AARCH64_SIMD_SPLIT)
-		      continue;
+		    /* AdvSIMD load patterns always have the memory operand
+		       (a DImode pointer) in the operand 1 position.  We
+		       want a const pointer to the element type in that
+		       position.  */
+		    gcc_assert (insn_data[d->code].operand[k].mode == DImode);
 
-		    if (is_load && k == 1)
+		    switch (d->mode)
 		      {
-			/* AdvSIMD load patterns always have the memory operand
-			   (a DImode pointer) in the operand 1 position.  We
-			   want a const pointer to the element type in that
-			   position.  */
-			gcc_assert (insn_data[icode].operand[k].mode ==
-				    DImode);
-
-			switch (1 << j)
-			  {
-			  case T_V8QI:
-			  case T_V16QI:
-			    eltype = const_intQI_pointer_node;
-			    break;
-
-			  case T_V4HI:
-			  case T_V8HI:
-			    eltype = const_intHI_pointer_node;
-			    break;
-
-			  case T_V2SI:
-			  case T_V4SI:
-			    eltype = const_intSI_pointer_node;
-			    break;
-
-			  case T_V2SF:
-			  case T_V4SF:
-			    eltype = const_float_pointer_node;
-			    break;
-
-			  case T_DI:
-			  case T_V2DI:
-			    eltype = const_intDI_pointer_node;
-			    break;
-
-			  case T_DF:
-			  case T_V2DF:
-			    eltype = const_double_pointer_node;
-			    break;
-
-			  default:
-			    gcc_unreachable ();
-			  }
+		      case T_V8QI:
+		      case T_V16QI:
+			eltype = const_intQI_pointer_node;
+			break;
+
+		      case T_V4HI:
+		      case T_V8HI:
+			eltype = const_intHI_pointer_node;
+			break;
+
+		      case T_V2SI:
+		      case T_V4SI:
+			eltype = const_intSI_pointer_node;
+			break;
+
+		      case T_V2SF:
+		      case T_V4SF:
+			eltype = const_float_pointer_node;
+			break;
+
+		      case T_DI:
+		      case T_V2DI:
+			eltype = const_intDI_pointer_node;
+			break;
+
+		      case T_DF:
+		      case T_V2DF:
+			eltype = const_double_pointer_node;
+			break;
+
+		      default:
+			gcc_unreachable ();
 		      }
-		    else if (is_store && k == 0)
+		  }
+		else if (is_store && k == 0)
+		  {
+		    /* Similarly, AdvSIMD store patterns use operand 0 as
+		       the memory location to store to (a DImode pointer).
+		       Use a pointer to the element type of the store in
+		       that position.  */
+		    gcc_assert (insn_data[d->code].operand[k].mode == DImode);
+
+		    switch (d->mode)
 		      {
-			/* Similarly, AdvSIMD store patterns use operand 0 as
-			   the memory location to store to (a DImode pointer).
-			   Use a pointer to the element type of the store in
-			   that position.  */
-			gcc_assert (insn_data[icode].operand[k].mode ==
-				    DImode);
-
-			switch (1 << j)
-			  {
-			  case T_V8QI:
-			  case T_V16QI:
-			    eltype = intQI_pointer_node;
-			    break;
-
-			  case T_V4HI:
-			  case T_V8HI:
-			    eltype = intHI_pointer_node;
-			    break;
-
-			  case T_V2SI:
-			  case T_V4SI:
-			    eltype = intSI_pointer_node;
-			    break;
-
-			  case T_V2SF:
-			  case T_V4SF:
-			    eltype = float_pointer_node;
-			    break;
-
-			  case T_DI:
-			  case T_V2DI:
-			    eltype = intDI_pointer_node;
-			    break;
-
-			  case T_DF:
-			  case T_V2DF:
-			    eltype = double_pointer_node;
-			    break;
-
-			  default:
-			    gcc_unreachable ();
-			  }
+		      case T_V8QI:
+		      case T_V16QI:
+			eltype = intQI_pointer_node;
+			break;
+
+		      case T_V4HI:
+		      case T_V8HI:
+			eltype = intHI_pointer_node;
+			break;
+
+		      case T_V2SI:
+		      case T_V4SI:
+			eltype = intSI_pointer_node;
+			break;
+
+		      case T_V2SF:
+		      case T_V4SF:
+			eltype = float_pointer_node;
+			break;
+
+		      case T_DI:
+		      case T_V2DI:
+			eltype = intDI_pointer_node;
+			break;
+
+		      case T_DF:
+		      case T_V2DF:
+			eltype = double_pointer_node;
+			break;
+
+		      default:
+			gcc_unreachable ();
 		      }
-		    else
+		  }
+		else
+		  {
+		    switch (insn_data[d->code].operand[k].mode)
 		      {
-			switch (insn_data[icode].operand[k].mode)
-			  {
-			  case VOIDmode:
-			    eltype = void_type_node;
-			    break;
-			    /* Scalars.  */
-			  case QImode:
-			    eltype = aarch64_simd_intQI_type_node;
-			    break;
-			  case HImode:
-			    eltype = aarch64_simd_intHI_type_node;
-			    break;
-			  case SImode:
-			    eltype = aarch64_simd_intSI_type_node;
-			    break;
-			  case SFmode:
-			    eltype = aarch64_simd_float_type_node;
-			    break;
-			  case DFmode:
-			    eltype = aarch64_simd_double_type_node;
-			    break;
-			  case DImode:
-			    eltype = aarch64_simd_intDI_type_node;
-			    break;
-			  case TImode:
-			    eltype = intTI_type_node;
-			    break;
-			  case EImode:
-			    eltype = intEI_type_node;
-			    break;
-			  case OImode:
-			    eltype = intOI_type_node;
-			    break;
-			  case CImode:
-			    eltype = intCI_type_node;
-			    break;
-			  case XImode:
-			    eltype = intXI_type_node;
-			    break;
-			    /* 64-bit vectors.  */
-			  case V8QImode:
-			    eltype = V8QI_type_node;
-			    break;
-			  case V4HImode:
-			    eltype = V4HI_type_node;
-			    break;
-			  case V2SImode:
-			    eltype = V2SI_type_node;
-			    break;
-			  case V2SFmode:
-			    eltype = V2SF_type_node;
-			    break;
-			    /* 128-bit vectors.  */
-			  case V16QImode:
-			    eltype = V16QI_type_node;
-			    break;
-			  case V8HImode:
-			    eltype = V8HI_type_node;
-			    break;
-			  case V4SImode:
-			    eltype = V4SI_type_node;
-			    break;
-			  case V4SFmode:
-			    eltype = V4SF_type_node;
-			    break;
-			  case V2DImode:
-			    eltype = V2DI_type_node;
-			    break;
-			  case V2DFmode:
-			    eltype = V2DF_type_node;
-			    break;
-			  default:
-			    gcc_unreachable ();
-			  }
+		      case VOIDmode:
+			eltype = void_type_node;
+			break;
+			/* Scalars.  */
+		      case QImode:
+			eltype = aarch64_simd_intQI_type_node;
+			break;
+		      case HImode:
+			eltype = aarch64_simd_intHI_type_node;
+			break;
+		      case SImode:
+			eltype = aarch64_simd_intSI_type_node;
+			break;
+		      case SFmode:
+			eltype = aarch64_simd_float_type_node;
+			break;
+		      case DFmode:
+			eltype = aarch64_simd_double_type_node;
+			break;
+		      case DImode:
+			eltype = aarch64_simd_intDI_type_node;
+			break;
+		      case TImode:
+			eltype = intTI_type_node;
+			break;
+		      case EImode:
+			eltype = intEI_type_node;
+			break;
+		      case OImode:
+			eltype = intOI_type_node;
+			break;
+		      case CImode:
+			eltype = intCI_type_node;
+			break;
+		      case XImode:
+			eltype = intXI_type_node;
+			break;
+			/* 64-bit vectors.  */
+		      case V8QImode:
+			eltype = V8QI_type_node;
+			break;
+		      case V4HImode:
+			eltype = V4HI_type_node;
+			break;
+		      case V2SImode:
+			eltype = V2SI_type_node;
+			break;
+		      case V2SFmode:
+			eltype = V2SF_type_node;
+			break;
+			/* 128-bit vectors.  */
+		      case V16QImode:
+			eltype = V16QI_type_node;
+			break;
+		      case V8HImode:
+			eltype = V8HI_type_node;
+			break;
+		      case V4SImode:
+			eltype = V4SI_type_node;
+			break;
+		      case V4SFmode:
+			eltype = V4SF_type_node;
+			break;
+		      case V2DImode:
+			eltype = V2DI_type_node;
+			break;
+		      case V2DFmode:
+			eltype = V2DF_type_node;
+			break;
+		      default:
+			gcc_unreachable ();
 		      }
-
-		    if (k == 0 && !is_store)
-		      return_type = eltype;
-		    else
-		      args = tree_cons (NULL_TREE, eltype, args);
 		  }
 
-		ftype = build_function_type (return_type, args);
+		if (k == 0 && !is_store)
+		  return_type = eltype;
+		else
+		  args = tree_cons (NULL_TREE, eltype, args);
 	      }
-	      break;
+	    ftype = build_function_type (return_type, args);
+	  }
+	  break;
 
-	    case AARCH64_SIMD_RESULTPAIR:
+	case AARCH64_SIMD_RESULTPAIR:
+	  {
+	    switch (insn_data[d->code].operand[1].mode)
 	      {
-		switch (insn_data[icode].operand[1].mode)
-		  {
-		  case V8QImode:
-		    ftype = void_ftype_pv8qi_v8qi_v8qi;
-		    break;
-		  case V4HImode:
-		    ftype = void_ftype_pv4hi_v4hi_v4hi;
-		    break;
-		  case V2SImode:
-		    ftype = void_ftype_pv2si_v2si_v2si;
-		    break;
-		  case V2SFmode:
-		    ftype = void_ftype_pv2sf_v2sf_v2sf;
-		    break;
-		  case DImode:
-		    ftype = void_ftype_pdi_di_di;
-		    break;
-		  case V16QImode:
-		    ftype = void_ftype_pv16qi_v16qi_v16qi;
-		    break;
-		  case V8HImode:
-		    ftype = void_ftype_pv8hi_v8hi_v8hi;
-		    break;
-		  case V4SImode:
-		    ftype = void_ftype_pv4si_v4si_v4si;
-		    break;
-		  case V4SFmode:
-		    ftype = void_ftype_pv4sf_v4sf_v4sf;
-		    break;
-		  case V2DImode:
-		    ftype = void_ftype_pv2di_v2di_v2di;
-		    break;
-		  case V2DFmode:
-		    ftype = void_ftype_pv2df_v2df_v2df;
-		    break;
-		  default:
-		    gcc_unreachable ();
-		  }
+	      case V8QImode:
+		ftype = void_ftype_pv8qi_v8qi_v8qi;
+		break;
+	      case V4HImode:
+		ftype = void_ftype_pv4hi_v4hi_v4hi;
+		break;
+	      case V2SImode:
+		ftype = void_ftype_pv2si_v2si_v2si;
+		break;
+	      case V2SFmode:
+		ftype = void_ftype_pv2sf_v2sf_v2sf;
+		break;
+	      case DImode:
+		ftype = void_ftype_pdi_di_di;
+		break;
+	      case V16QImode:
+		ftype = void_ftype_pv16qi_v16qi_v16qi;
+		break;
+	      case V8HImode:
+		ftype = void_ftype_pv8hi_v8hi_v8hi;
+		break;
+	      case V4SImode:
+		ftype = void_ftype_pv4si_v4si_v4si;
+		break;
+	      case V4SFmode:
+		ftype = void_ftype_pv4sf_v4sf_v4sf;
+		break;
+	      case V2DImode:
+		ftype = void_ftype_pv2di_v2di_v2di;
+		break;
+	      case V2DFmode:
+		ftype = void_ftype_pv2df_v2df_v2df;
+		break;
+	      default:
+		gcc_unreachable ();
 	      }
-	      break;
-
-	    case AARCH64_SIMD_REINTERP:
+	  }
+	  break;
+
+	case AARCH64_SIMD_REINTERP:
+	  {
+	    /* We iterate over 6 doubleword types, then 6 quadword
+	       types.  */
+	    int rhs_d = d->mode % NUM_DREG_TYPES;
+	    int rhs_q = (d->mode - NUM_DREG_TYPES) % NUM_QREG_TYPES;
+	    switch (insn_data[d->code].operand[0].mode)
 	      {
-		/* We iterate over 6 doubleword types, then 6 quadword
-		   types.  */
-		int rhs_d = j % NUM_DREG_TYPES;
-		int rhs_q = (j - NUM_DREG_TYPES) % NUM_QREG_TYPES;
-		switch (insn_data[icode].operand[0].mode)
-		  {
-		  case V8QImode:
-		    ftype = reinterp_ftype_dreg[0][rhs_d];
-		    break;
-		  case V4HImode:
-		    ftype = reinterp_ftype_dreg[1][rhs_d];
-		    break;
-		  case V2SImode:
-		    ftype = reinterp_ftype_dreg[2][rhs_d];
-		    break;
-		  case V2SFmode:
-		    ftype = reinterp_ftype_dreg[3][rhs_d];
-		    break;
-		  case DImode:
-		    ftype = reinterp_ftype_dreg[4][rhs_d];
-		    break;
-		  case DFmode:
-		    ftype = reinterp_ftype_dreg[5][rhs_d];
-		    break;
-		  case V16QImode:
-		    ftype = reinterp_ftype_qreg[0][rhs_q];
-		    break;
-		  case V8HImode:
-		    ftype = reinterp_ftype_qreg[1][rhs_q];
-		    break;
-		  case V4SImode:
-		    ftype = reinterp_ftype_qreg[2][rhs_q];
-		    break;
-		  case V4SFmode:
-		    ftype = reinterp_ftype_qreg[3][rhs_q];
-		    break;
-		  case V2DImode:
-		    ftype = reinterp_ftype_qreg[4][rhs_q];
-		    break;
-		  case V2DFmode:
-		    ftype = reinterp_ftype_qreg[5][rhs_q];
-		    break;
-		  default:
-		    gcc_unreachable ();
-		  }
+	      case V8QImode:
+		ftype = reinterp_ftype_dreg[0][rhs_d];
+		break;
+	      case V4HImode:
+		ftype = reinterp_ftype_dreg[1][rhs_d];
+		break;
+	      case V2SImode:
+		ftype = reinterp_ftype_dreg[2][rhs_d];
+		break;
+	      case V2SFmode:
+		ftype = reinterp_ftype_dreg[3][rhs_d];
+		break;
+	      case DImode:
+		ftype = reinterp_ftype_dreg[4][rhs_d];
+		break;
+	      case DFmode:
+		ftype = reinterp_ftype_dreg[5][rhs_d];
+		break;
+	      case V16QImode:
+		ftype = reinterp_ftype_qreg[0][rhs_q];
+		break;
+	      case V8HImode:
+		ftype = reinterp_ftype_qreg[1][rhs_q];
+		break;
+	      case V4SImode:
+		ftype = reinterp_ftype_qreg[2][rhs_q];
+		break;
+	      case V4SFmode:
+		ftype = reinterp_ftype_qreg[3][rhs_q];
+		break;
+	      case V2DImode:
+		ftype = reinterp_ftype_qreg[4][rhs_q];
+		break;
+	      case V2DFmode:
+		ftype = reinterp_ftype_qreg[5][rhs_q];
+		break;
+	      default:
+		gcc_unreachable ();
 	      }
-	      break;
+	  }
+	  break;
 
-	    default:
-	      gcc_unreachable ();
-	    }
-
-	  gcc_assert (ftype != NULL);
+	default:
+	  gcc_unreachable ();
+	}
+      gcc_assert (ftype != NULL);
 
-	  snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s%s",
-		    d->name, modenames[j]);
+      snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s%s",
+		d->name, modenames[d->mode]);
 
-	  add_builtin_function (namebuf, ftype, fcode++, BUILT_IN_MD, NULL,
-				NULL_TREE);
-	}
+      fndecl = add_builtin_function (namebuf, ftype, fcode, BUILT_IN_MD,
+				     NULL, NULL_TREE);
+      aarch64_builtin_decls[fcode] = fndecl;
     }
 }
 
-static int
-aarch64_simd_builtin_compare (const void *a, const void *b)
+void
+aarch64_init_builtins (void)
 {
-  const aarch64_simd_builtin_datum *const key =
-    (const aarch64_simd_builtin_datum *) a;
-  const aarch64_simd_builtin_datum *const memb =
-    (const aarch64_simd_builtin_datum *) b;
-  unsigned int soughtcode = key->base_fcode;
-
-  if (soughtcode >= memb->base_fcode
-      && soughtcode < memb->base_fcode + memb->num_vars)
-    return 0;
-  else if (soughtcode < memb->base_fcode)
-    return -1;
-  else
-    return 1;
+  tree ftype, decl = NULL;
+
+  ftype = build_function_type (ptr_type_node, void_list_node);
+  decl = add_builtin_function ("__builtin_thread_pointer", ftype,
+			       AARCH64_BUILTIN_THREAD_POINTER, BUILT_IN_MD,
+			       NULL, NULL_TREE);
+  TREE_NOTHROW (decl) = 1;
+  TREE_READONLY (decl) = 1;
+  aarch64_builtin_decls[AARCH64_BUILTIN_THREAD_POINTER] = decl;
+
+  if (TARGET_SIMD)
+    aarch64_init_simd_builtins ();
 }
 
-
-static enum insn_code
-locate_simd_builtin_icode (int fcode, aarch64_simd_itype * itype)
+tree
+aarch64_builtin_decl (unsigned code, bool initialize_p ATTRIBUTE_UNUSED)
 {
-  aarch64_simd_builtin_datum key
-    = { NULL, (aarch64_simd_itype) 0, 0, {CODE_FOR_nothing}, 0, 0};
-  aarch64_simd_builtin_datum *found;
-  int idx;
-
-  key.base_fcode = fcode;
-  found = (aarch64_simd_builtin_datum *)
-    bsearch (&key, &aarch64_simd_builtin_data[0],
-	     ARRAY_SIZE (aarch64_simd_builtin_data),
-	     sizeof (aarch64_simd_builtin_data[0]),
-	     aarch64_simd_builtin_compare);
-  gcc_assert (found);
-  idx = fcode - (int) found->base_fcode;
-  gcc_assert (idx >= 0 && idx < T_MAX && idx < (int) found->num_vars);
-
-  if (itype)
-    *itype = found->itype;
-
-  return found->codes[idx];
+  if (code >= AARCH64_BUILTIN_MAX)
+    return error_mark_node;
+
+  return aarch64_builtin_decls[code];
 }
 
 typedef enum
@@ -1225,8 +1119,10 @@ aarch64_simd_expand_args (rtx target, int icode, int have_retval,
 rtx
 aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
 {
-  aarch64_simd_itype itype;
-  enum insn_code icode = locate_simd_builtin_icode (fcode, &itype);
+  aarch64_simd_builtin_datum *d =
+		&aarch64_simd_builtin_data[fcode - (AARCH64_SIMD_BUILTIN_BASE + 1)];
+  aarch64_simd_itype itype = d->itype;
+  enum insn_code icode = d->code;
 
   switch (itype)
     {
@@ -1318,3 +1214,24 @@ aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
       gcc_unreachable ();
     }
 }
+
+/* Expand an expression EXP that calls a built-in function,
+   with result going to TARGET if that's convenient.  */
+rtx
+aarch64_expand_builtin (tree exp,
+		     rtx target,
+		     rtx subtarget ATTRIBUTE_UNUSED,
+		     enum machine_mode mode ATTRIBUTE_UNUSED,
+		     int ignore ATTRIBUTE_UNUSED)
+{
+  tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
+  int fcode = DECL_FUNCTION_CODE (fndecl);
+
+  if (fcode == AARCH64_BUILTIN_THREAD_POINTER)
+    return aarch64_load_tp (target);
+
+  if (fcode >= AARCH64_SIMD_BUILTIN_BASE)
+    return aarch64_simd_expand_builtin (fcode, exp, target);
+
+  return NULL_RTX;
+}
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 11e35103657..31d978b4c86 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -227,4 +227,13 @@ void aarch64_split_atomic_op (enum rtx_code, rtx, rtx, rtx, rtx, rtx, rtx);
 
 #endif /* RTX_CODE */
 
+rtx aarch64_load_tp (rtx target);
+void aarch64_init_builtins (void);
+rtx aarch64_expand_builtin (tree exp,
+			    rtx target,
+			    rtx subtarget ATTRIBUTE_UNUSED,
+			    enum machine_mode mode ATTRIBUTE_UNUSED,
+			    int ignore ATTRIBUTE_UNUSED);
+tree aarch64_builtin_decl (unsigned, bool ATTRIBUTE_UNUSED);
+
 #endif /* GCC_AARCH64_PROTOS_H */
diff --git a/gcc/config/aarch64/aarch64-simd-builtins.def b/gcc/config/aarch64/aarch64-simd-builtins.def
new file mode 100644
index 00000000000..2e3c4e1ca5f
--- /dev/null
+++ b/gcc/config/aarch64/aarch64-simd-builtins.def
@@ -0,0 +1,208 @@
+
+/* In the list below, the BUILTIN_<ITERATOR> macros should
+   correspond to the iterator used to construct the instruction's
+   patterns in aarch64-simd.md.  A helpful idiom to follow when
+   adding new builtins is to add a line for each pattern in the md
+   file.  Thus, ADDP, which has one pattern defined for the VD_BHSI
+   iterator, and one for DImode, has two entries below.  */
+
+  BUILTIN_VD_RE (CREATE, create)
+  BUILTIN_VQ_S (GETLANE, get_lane_signed)
+  BUILTIN_VDQ (GETLANE, get_lane_unsigned)
+  BUILTIN_VDQF (GETLANE, get_lane)
+  VAR1 (GETLANE, get_lane, di)
+  BUILTIN_VDC (COMBINE, combine)
+  BUILTIN_VB (BINOP, pmul)
+  BUILTIN_VDQF (UNOP, sqrt)
+  BUILTIN_VD_BHSI (BINOP, addp)
+  VAR1 (UNOP, addp, di)
+
+  BUILTIN_VD_RE (REINTERP, reinterpretdi)
+  BUILTIN_VDC (REINTERP, reinterpretv8qi)
+  BUILTIN_VDC (REINTERP, reinterpretv4hi)
+  BUILTIN_VDC (REINTERP, reinterpretv2si)
+  BUILTIN_VDC (REINTERP, reinterpretv2sf)
+  BUILTIN_VQ (REINTERP, reinterpretv16qi)
+  BUILTIN_VQ (REINTERP, reinterpretv8hi)
+  BUILTIN_VQ (REINTERP, reinterpretv4si)
+  BUILTIN_VQ (REINTERP, reinterpretv4sf)
+  BUILTIN_VQ (REINTERP, reinterpretv2di)
+  BUILTIN_VQ (REINTERP, reinterpretv2df)
+
+  BUILTIN_VDQ_I (BINOP, dup_lane)
+  BUILTIN_SDQ_I (BINOP, dup_lane)
+  /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
+  BUILTIN_VSDQ_I (BINOP, sqshl)
+  BUILTIN_VSDQ_I (BINOP, uqshl)
+  BUILTIN_VSDQ_I (BINOP, sqrshl)
+  BUILTIN_VSDQ_I (BINOP, uqrshl)
+  /* Implemented by aarch64_<su_optab><optab><mode>.  */
+  BUILTIN_VSDQ_I (BINOP, sqadd)
+  BUILTIN_VSDQ_I (BINOP, uqadd)
+  BUILTIN_VSDQ_I (BINOP, sqsub)
+  BUILTIN_VSDQ_I (BINOP, uqsub)
+  /* Implemented by aarch64_<sur>qadd<mode>.  */
+  BUILTIN_VSDQ_I (BINOP, suqadd)
+  BUILTIN_VSDQ_I (BINOP, usqadd)
+
+  /* Implemented by aarch64_get_dreg<VSTRUCT:mode><VDC:mode>.  */
+  BUILTIN_VDC (GETLANE, get_dregoi)
+  BUILTIN_VDC (GETLANE, get_dregci)
+  BUILTIN_VDC (GETLANE, get_dregxi)
+  /* Implemented by aarch64_get_qreg<VSTRUCT:mode><VQ:mode>.  */
+  BUILTIN_VQ (GETLANE, get_qregoi)
+  BUILTIN_VQ (GETLANE, get_qregci)
+  BUILTIN_VQ (GETLANE, get_qregxi)
+  /* Implemented by aarch64_set_qreg<VSTRUCT:mode><VQ:mode>.  */
+  BUILTIN_VQ (SETLANE, set_qregoi)
+  BUILTIN_VQ (SETLANE, set_qregci)
+  BUILTIN_VQ (SETLANE, set_qregxi)
+  /* Implemented by aarch64_ld<VSTRUCT:nregs><VDC:mode>.  */
+  BUILTIN_VDC (LOADSTRUCT, ld2)
+  BUILTIN_VDC (LOADSTRUCT, ld3)
+  BUILTIN_VDC (LOADSTRUCT, ld4)
+  /* Implemented by aarch64_ld<VSTRUCT:nregs><VQ:mode>.  */
+  BUILTIN_VQ (LOADSTRUCT, ld2)
+  BUILTIN_VQ (LOADSTRUCT, ld3)
+  BUILTIN_VQ (LOADSTRUCT, ld4)
+  /* Implemented by aarch64_st<VSTRUCT:nregs><VDC:mode>.  */
+  BUILTIN_VDC (STORESTRUCT, st2)
+  BUILTIN_VDC (STORESTRUCT, st3)
+  BUILTIN_VDC (STORESTRUCT, st4)
+  /* Implemented by aarch64_st<VSTRUCT:nregs><VQ:mode>.  */
+  BUILTIN_VQ (STORESTRUCT, st2)
+  BUILTIN_VQ (STORESTRUCT, st3)
+  BUILTIN_VQ (STORESTRUCT, st4)
+
+  BUILTIN_VQW (BINOP, saddl2)
+  BUILTIN_VQW (BINOP, uaddl2)
+  BUILTIN_VQW (BINOP, ssubl2)
+  BUILTIN_VQW (BINOP, usubl2)
+  BUILTIN_VQW (BINOP, saddw2)
+  BUILTIN_VQW (BINOP, uaddw2)
+  BUILTIN_VQW (BINOP, ssubw2)
+  BUILTIN_VQW (BINOP, usubw2)
+  /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
+  BUILTIN_VDW (BINOP, saddl)
+  BUILTIN_VDW (BINOP, uaddl)
+  BUILTIN_VDW (BINOP, ssubl)
+  BUILTIN_VDW (BINOP, usubl)
+  /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
+  BUILTIN_VDW (BINOP, saddw)
+  BUILTIN_VDW (BINOP, uaddw)
+  BUILTIN_VDW (BINOP, ssubw)
+  BUILTIN_VDW (BINOP, usubw)
+  /* Implemented by aarch64_<sur>h<addsub><mode>.  */
+  BUILTIN_VQ_S (BINOP, shadd)
+  BUILTIN_VQ_S (BINOP, uhadd)
+  BUILTIN_VQ_S (BINOP, srhadd)
+  BUILTIN_VQ_S (BINOP, urhadd)
+  /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
+  BUILTIN_VQN (BINOP, addhn)
+  BUILTIN_VQN (BINOP, raddhn)
+  /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
+  BUILTIN_VQN (TERNOP, addhn2)
+  BUILTIN_VQN (TERNOP, raddhn2)
+
+  BUILTIN_VSQN_HSDI (UNOP, sqmovun)
+  /* Implemented by aarch64_<sur>qmovn<mode>.  */
+  BUILTIN_VSQN_HSDI (UNOP, sqmovn)
+  BUILTIN_VSQN_HSDI (UNOP, uqmovn)
+  /* Implemented by aarch64_s<optab><mode>.  */
+  BUILTIN_VSDQ_I_BHSI (UNOP, sqabs)
+  BUILTIN_VSDQ_I_BHSI (UNOP, sqneg)
+
+  BUILTIN_VSD_HSI (QUADOP, sqdmlal_lane)
+  BUILTIN_VSD_HSI (QUADOP, sqdmlsl_lane)
+  BUILTIN_VSD_HSI (QUADOP, sqdmlal_laneq)
+  BUILTIN_VSD_HSI (QUADOP, sqdmlsl_laneq)
+  BUILTIN_VQ_HSI (TERNOP, sqdmlal2)
+  BUILTIN_VQ_HSI (TERNOP, sqdmlsl2)
+  BUILTIN_VQ_HSI (QUADOP, sqdmlal2_lane)
+  BUILTIN_VQ_HSI (QUADOP, sqdmlsl2_lane)
+  BUILTIN_VQ_HSI (QUADOP, sqdmlal2_laneq)
+  BUILTIN_VQ_HSI (QUADOP, sqdmlsl2_laneq)
+  BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n)
+  BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n)
+  /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
+  BUILTIN_VSD_HSI (TERNOP, sqdmlal)
+  BUILTIN_VSD_HSI (TERNOP, sqdmlsl)
+  /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
+  BUILTIN_VD_HSI (TERNOP, sqdmlal_n)
+  BUILTIN_VD_HSI (TERNOP, sqdmlsl_n)
+
+  BUILTIN_VSD_HSI (BINOP, sqdmull)
+  BUILTIN_VSD_HSI (TERNOP, sqdmull_lane)
+  BUILTIN_VD_HSI (TERNOP, sqdmull_laneq)
+  BUILTIN_VD_HSI (BINOP, sqdmull_n)
+  BUILTIN_VQ_HSI (BINOP, sqdmull2)
+  BUILTIN_VQ_HSI (TERNOP, sqdmull2_lane)
+  BUILTIN_VQ_HSI (TERNOP, sqdmull2_laneq)
+  BUILTIN_VQ_HSI (BINOP, sqdmull2_n)
+  /* Implemented by aarch64_sq<r>dmulh<mode>.  */
+  BUILTIN_VSDQ_HSI (BINOP, sqdmulh)
+  BUILTIN_VSDQ_HSI (BINOP, sqrdmulh)
+  /* Implemented by aarch64_sq<r>dmulh_lane<mode>.  */
+  BUILTIN_VSDQ_HSI (TERNOP, sqdmulh_lane)
+  BUILTIN_VSDQ_HSI (TERNOP, sqrdmulh_lane)
+
+  BUILTIN_VSDQ_I_DI (BINOP, sshl_n)
+  BUILTIN_VSDQ_I_DI (BINOP, ushl_n)
+  /* Implemented by aarch64_<sur>shl<mode>.  */
+  BUILTIN_VSDQ_I_DI (BINOP, sshl)
+  BUILTIN_VSDQ_I_DI (BINOP, ushl)
+  BUILTIN_VSDQ_I_DI (BINOP, srshl)
+  BUILTIN_VSDQ_I_DI (BINOP, urshl)
+
+  BUILTIN_VSDQ_I_DI (SHIFTIMM, sshr_n)
+  BUILTIN_VSDQ_I_DI (SHIFTIMM, ushr_n)
+  /* Implemented by aarch64_<sur>shr_n<mode>.  */
+  BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n)
+  BUILTIN_VSDQ_I_DI (SHIFTIMM, urshr_n)
+  /* Implemented by aarch64_<sur>sra_n<mode>.  */
+  BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n)
+  BUILTIN_VSDQ_I_DI (SHIFTACC, usra_n)
+  BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n)
+  BUILTIN_VSDQ_I_DI (SHIFTACC, ursra_n)
+  /* Implemented by aarch64_<sur>shll_n<mode>.  */
+  BUILTIN_VDW (SHIFTIMM, sshll_n)
+  BUILTIN_VDW (SHIFTIMM, ushll_n)
+  /* Implemented by aarch64_<sur>shll2_n<mode>.  */
+  BUILTIN_VQW (SHIFTIMM, sshll2_n)
+  BUILTIN_VQW (SHIFTIMM, ushll2_n)
+  /* Implemented by aarch64_<sur>q<r>shr<u>n_n<mode>.  */
+  BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrun_n)
+  BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrun_n)
+  BUILTIN_VSQN_HSDI (SHIFTIMM, sqshrn_n)
+  BUILTIN_VSQN_HSDI (SHIFTIMM, uqshrn_n)
+  BUILTIN_VSQN_HSDI (SHIFTIMM, sqrshrn_n)
+  BUILTIN_VSQN_HSDI (SHIFTIMM, uqrshrn_n)
+  /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
+  BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n)
+  BUILTIN_VSDQ_I_DI (SHIFTINSERT, usri_n)
+  BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n)
+  BUILTIN_VSDQ_I_DI (SHIFTINSERT, usli_n)
+  /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
+  BUILTIN_VSDQ_I (SHIFTIMM, sqshlu_n)
+  BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n)
+  BUILTIN_VSDQ_I (SHIFTIMM, uqshl_n)
+
+  /* Implemented by aarch64_cm<cmp><mode>.  */
+  BUILTIN_VSDQ_I_DI (BINOP, cmeq)
+  BUILTIN_VSDQ_I_DI (BINOP, cmge)
+  BUILTIN_VSDQ_I_DI (BINOP, cmgt)
+  BUILTIN_VSDQ_I_DI (BINOP, cmle)
+  BUILTIN_VSDQ_I_DI (BINOP, cmlt)
+  /* Implemented by aarch64_cm<cmp><mode>.  */
+  BUILTIN_VSDQ_I_DI (BINOP, cmhs)
+  BUILTIN_VSDQ_I_DI (BINOP, cmhi)
+  BUILTIN_VSDQ_I_DI (BINOP, cmtst)
+
+  /* Implemented by aarch64_<fmaxmin><mode>.  */
+  BUILTIN_VDQF (BINOP, fmax)
+  BUILTIN_VDQF (BINOP, fmin)
+  /* Implemented by aarch64_<maxmin><mode>.  */
+  BUILTIN_VDQ_BHSI (BINOP, smax)
+  BUILTIN_VDQ_BHSI (BINOP, smin)
+  BUILTIN_VDQ_BHSI (BINOP, umax)
+  BUILTIN_VDQ_BHSI (BINOP, umin)
diff --git a/gcc/config/aarch64/aarch64-simd.md b/gcc/config/aarch64/aarch64-simd.md
index a7ddfb1c1d3..b3d01c19199 100644
--- a/gcc/config/aarch64/aarch64-simd.md
+++ b/gcc/config/aarch64/aarch64-simd.md
@@ -1467,6 +1467,150 @@
    (set_attr "simd_mode" "V2SI")]
 )
 
+;; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
+;; allocation.  For an intrinsic of form:
+;;   vD = bsl_* (vS, vN, vM)
+;; We can use any of:
+;;   bsl vS, vN, vM  (if D = S)
+;;   bit vD, vN, vS  (if D = M, so 1-bits in vS choose bits from vN, else vM)
+;;   bif vD, vM, vS  (if D = N, so 0-bits in vS choose bits from vM, else vN)
+
+(define_insn "aarch64_simd_bsl<mode>_internal"
+  [(set (match_operand:VDQ 0 "register_operand"		     "=w,w,w")
+	(unspec:VDQ [(match_operand:VDQ 1 "register_operand" " 0,w,w")
+		     (match_operand:VDQ 2 "register_operand" " w,w,0")
+		     (match_operand:VDQ 3 "register_operand" " w,0,w")]
+		    UNSPEC_BSL))]
+  "TARGET_SIMD"
+  "@
+  bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
+  bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
+  bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
+)
+
+(define_expand "aarch64_simd_bsl<mode>"
+  [(set (match_operand:VDQ 0 "register_operand")
+	(unspec:VDQ [(match_operand:<V_cmp_result> 1 "register_operand")
+		     (match_operand:VDQ 2 "register_operand")
+		     (match_operand:VDQ 3 "register_operand")]
+		    UNSPEC_BSL))]
+  "TARGET_SIMD"
+{
+  /* We can't alias operands together if they have different modes.  */
+  operands[1] = gen_lowpart (<MODE>mode, operands[1]);
+})
+
+(define_expand "aarch64_vcond_internal<mode>"
+  [(set (match_operand:VDQ 0 "register_operand")
+	(if_then_else:VDQ
+	  (match_operator 3 "comparison_operator"
+	    [(match_operand:VDQ 4 "register_operand")
+	     (match_operand:VDQ 5 "nonmemory_operand")])
+	  (match_operand:VDQ 1 "register_operand")
+	  (match_operand:VDQ 2 "register_operand")))]
+  "TARGET_SIMD"
+{
+  int inverse = 0, has_zero_imm_form = 0;
+  rtx mask = gen_reg_rtx (<MODE>mode);
+
+  switch (GET_CODE (operands[3]))
+    {
+    case LE:
+    case LT:
+    case NE:
+      inverse = 1;
+      /* Fall through.  */
+    case GE:
+    case GT:
+    case EQ:
+      has_zero_imm_form = 1;
+      break;
+    case LEU:
+    case LTU:
+      inverse = 1;
+      break;
+    default:
+      break;
+    }
+
+  if (!REG_P (operands[5])
+      && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form))
+    operands[5] = force_reg (<MODE>mode, operands[5]);
+
+  switch (GET_CODE (operands[3]))
+    {
+    case LT:
+    case GE:
+      emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
+      break;
+
+    case LE:
+    case GT:
+      emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
+      break;
+
+    case LTU:
+    case GEU:
+      emit_insn (gen_aarch64_cmhs<mode> (mask, operands[4], operands[5]));
+      break;
+
+    case LEU:
+    case GTU:
+      emit_insn (gen_aarch64_cmhi<mode> (mask, operands[4], operands[5]));
+      break;
+
+    case NE:
+    case EQ:
+      emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
+      break;
+
+    default:
+      gcc_unreachable ();
+    }
+
+  if (inverse)
+    emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[2],
+				    operands[1]));
+  else
+    emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[1],
+				    operands[2]));
+
+  DONE;
+})
+
+(define_expand "vcond<mode><mode>"
+  [(set (match_operand:VDQ 0 "register_operand")
+	(if_then_else:VDQ
+	  (match_operator 3 "comparison_operator"
+	    [(match_operand:VDQ 4 "register_operand")
+	     (match_operand:VDQ 5 "nonmemory_operand")])
+	  (match_operand:VDQ 1 "register_operand")
+	  (match_operand:VDQ 2 "register_operand")))]
+  "TARGET_SIMD"
+{
+  emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
+					       operands[2], operands[3],
+					       operands[4], operands[5]));
+  DONE;
+})
+
+
+(define_expand "vcondu<mode><mode>"
+  [(set (match_operand:VDQ 0 "register_operand")
+	(if_then_else:VDQ
+	  (match_operator 3 "comparison_operator"
+	    [(match_operand:VDQ 4 "register_operand")
+	     (match_operand:VDQ 5 "nonmemory_operand")])
+	  (match_operand:VDQ 1 "register_operand")
+	  (match_operand:VDQ 2 "register_operand")))]
+  "TARGET_SIMD"
+{
+  emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
+					       operands[2], operands[3],
+					       operands[4], operands[5]));
+  DONE;
+})
+
 ;; Patterns for AArch64 SIMD Intrinsics.
 
 (define_expand "aarch64_create<mode>"
diff --git a/gcc/config/aarch64/aarch64.c b/gcc/config/aarch64/aarch64.c
index 212bf116802..35f4b9267c8 100644
--- a/gcc/config/aarch64/aarch64.c
+++ b/gcc/config/aarch64/aarch64.c
@@ -102,7 +102,6 @@ static bool aarch64_vfp_is_call_or_return_candidate (enum machine_mode,
 						     bool *);
 static void aarch64_elf_asm_constructor (rtx, int) ATTRIBUTE_UNUSED;
 static void aarch64_elf_asm_destructor (rtx, int) ATTRIBUTE_UNUSED;
-static rtx aarch64_load_tp (rtx);
 static void aarch64_override_options_after_change (void);
 static int aarch64_simd_valid_immediate (rtx, enum machine_mode, int, rtx *,
 					 int *, unsigned char *, int *, int *);
@@ -2167,12 +2166,10 @@ aarch64_final_eh_return_addr (void)
 
 /* Output code to build up a constant in a register.  */
 static void
-aarch64_build_constant (FILE *file,
-			int regnum,
-			HOST_WIDE_INT val)
+aarch64_build_constant (int regnum, HOST_WIDE_INT val)
 {
   if (aarch64_bitmask_imm (val, DImode))
-    asm_fprintf (file, "\tmovi\t%r, %wd\n", regnum, val);
+    emit_move_insn (gen_rtx_REG (Pmode, regnum), GEN_INT (val));
   else
     {
       int i;
@@ -2203,12 +2200,14 @@ aarch64_build_constant (FILE *file,
 	 the same.  */
       if (ncount < zcount)
 	{
-	  asm_fprintf (file, "\tmovn\t%r, %wd\n", regnum, (~val) & 0xffff);
+	  emit_move_insn (gen_rtx_REG (Pmode, regnum),
+			  GEN_INT ((~val) & 0xffff));
 	  tval = 0xffff;
 	}
       else
 	{
-	  asm_fprintf (file, "\tmovz\t%r, %wd\n", regnum, val & 0xffff);
+	  emit_move_insn (gen_rtx_REG (Pmode, regnum),
+			  GEN_INT (val & 0xffff));
 	  tval = 0;
 	}
 
@@ -2217,39 +2216,47 @@ aarch64_build_constant (FILE *file,
       for (i = 16; i < 64; i += 16)
 	{
 	  if ((val & 0xffff) != tval)
-	    asm_fprintf (file, "\tmovk\t%r, %wd, lsl %d\n",
-			 regnum, val & 0xffff, i);
+	    emit_insn (gen_insv_immdi (gen_rtx_REG (Pmode, regnum),
+				       GEN_INT (i), GEN_INT (val & 0xffff)));
 	  val >>= 16;
 	}
     }
 }
 
 static void
-aarch64_add_constant (FILE *file, int regnum, int scratchreg,
-			     HOST_WIDE_INT delta)
+aarch64_add_constant (int regnum, int scratchreg, HOST_WIDE_INT delta)
 {
   HOST_WIDE_INT mdelta = delta;
+  rtx this_rtx = gen_rtx_REG (Pmode, regnum);
+  rtx scratch_rtx = gen_rtx_REG (Pmode, scratchreg);
 
   if (mdelta < 0)
     mdelta = -mdelta;
 
   if (mdelta >= 4096 * 4096)
     {
-      aarch64_build_constant (file, scratchreg, delta);
-      asm_fprintf (file, "\tadd\t%r, %r, %r\n", regnum, regnum,
-		   scratchreg);
+      aarch64_build_constant (scratchreg, delta);
+      emit_insn (gen_add3_insn (this_rtx, this_rtx, scratch_rtx));
     }
   else if (mdelta > 0)
     {
-      const char *const mi_op = delta < 0 ? "sub" : "add";
-
       if (mdelta >= 4096)
-	asm_fprintf (file, "\t%s\t%r, %r, %wd, lsl 12\n", mi_op, regnum, regnum,
-		     mdelta / 4096);
-
+	{
+	  emit_insn (gen_rtx_SET (Pmode, scratch_rtx, GEN_INT (mdelta / 4096)));
+	  rtx shift = gen_rtx_ASHIFT (Pmode, scratch_rtx, GEN_INT (12));
+	  if (delta < 0)
+	    emit_insn (gen_rtx_SET (Pmode, this_rtx,
+				    gen_rtx_MINUS (Pmode, this_rtx, shift)));
+	  else
+	    emit_insn (gen_rtx_SET (Pmode, this_rtx,
+				    gen_rtx_PLUS (Pmode, this_rtx, shift)));
+	}
       if (mdelta % 4096 != 0)
-	asm_fprintf (file, "\t%s\t%r, %r, %wd\n", mi_op, regnum, regnum,
-		     mdelta % 4096);
+	{
+	  scratch_rtx = GEN_INT ((delta < 0 ? -1 : 1) * (mdelta % 4096));
+	  emit_insn (gen_rtx_SET (Pmode, this_rtx,
+				  gen_rtx_PLUS (Pmode, this_rtx, scratch_rtx)));
+	}
     }
 }
 
@@ -2272,7 +2279,7 @@ aarch64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
   emit_note (NOTE_INSN_PROLOGUE_END);
 
   if (vcall_offset == 0)
-    aarch64_add_constant (file, this_regno, IP1_REGNUM, delta);
+    aarch64_add_constant (this_regno, IP1_REGNUM, delta);
   else
     {
       gcc_assert ((vcall_offset & 0x7) == 0);
@@ -2288,7 +2295,7 @@ aarch64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
 	    addr = gen_rtx_PRE_MODIFY (Pmode, this_rtx,
 				       plus_constant (this_rtx, delta));
 	  else
-	    aarch64_add_constant (file, this_regno, IP1_REGNUM, delta);
+	    aarch64_add_constant (this_regno, IP1_REGNUM, delta);
 	}
 
       aarch64_emit_move (temp0, gen_rtx_MEM (Pmode, addr));
@@ -2297,7 +2304,7 @@ aarch64_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED,
 	  addr = plus_constant (temp0, vcall_offset);
       else
 	{
-	  aarch64_build_constant (file, IP1_REGNUM, vcall_offset);
+	  aarch64_build_constant (IP1_REGNUM, vcall_offset);
 	  addr = gen_rtx_PLUS (Pmode, temp0, temp1);
 	}
 
@@ -5003,23 +5010,7 @@ aarch64_legitimate_constant_p (enum machine_mode mode, rtx x)
   return aarch64_constant_address_p (x);
 }
 
-static void
-aarch64_init_builtins (void)
-{
-  tree ftype, decl = NULL;
-
-  ftype = build_function_type (ptr_type_node, void_list_node);
-  decl = add_builtin_function ("__builtin_thread_pointer", ftype,
-			       AARCH64_BUILTIN_THREAD_POINTER, BUILT_IN_MD,
-			       NULL, NULL_TREE);
-  TREE_NOTHROW (decl) = 1;
-  TREE_READONLY (decl) = 1;
-
-  if (TARGET_SIMD)
-    init_aarch64_simd_builtins ();
-}
-
-static rtx
+rtx
 aarch64_load_tp (rtx target)
 {
   if (!target
@@ -5032,27 +5023,6 @@ aarch64_load_tp (rtx target)
   return target;
 }
 
-/* Expand an expression EXP that calls a built-in function,
-   with result going to TARGET if that's convenient.  */
-static rtx
-aarch64_expand_builtin (tree exp,
-		     rtx target,
-		     rtx subtarget ATTRIBUTE_UNUSED,
-		     enum machine_mode mode ATTRIBUTE_UNUSED,
-		     int ignore ATTRIBUTE_UNUSED)
-{
-  tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
-  int fcode = DECL_FUNCTION_CODE (fndecl);
-
-  if (fcode == AARCH64_BUILTIN_THREAD_POINTER)
-    return aarch64_load_tp (target);
-
-  if (fcode >= AARCH64_SIMD_BUILTIN_BASE)
-    return aarch64_simd_expand_builtin (fcode, exp, target);
-
-  return NULL_RTX;
-}
-
 /* On AAPCS systems, this is the "struct __va_list".  */
 static GTY(()) tree va_list_type;
 
@@ -5085,6 +5055,7 @@ aarch64_build_builtin_va_list (void)
 			     va_list_type);
   DECL_ARTIFICIAL (va_list_name) = 1;
   TYPE_NAME (va_list_type) = va_list_name;
+  TYPE_STUB_DECL (va_list_type) = va_list_name;
 
   /* Create the fields.  */
   f_stack = build_decl (BUILTINS_LOCATION,
@@ -6340,6 +6311,31 @@ aarch64_simd_attr_length_move (rtx insn)
   return 4;
 }
 
+/* Implement target hook TARGET_VECTOR_ALIGNMENT.  The AAPCS64 sets the maximum
+   alignment of a vector to 128 bits.  */
+static HOST_WIDE_INT
+aarch64_simd_vector_alignment (const_tree type)
+{
+  HOST_WIDE_INT align = tree_low_cst (TYPE_SIZE (type), 0);
+  return MIN (align, 128);
+}
+
+/* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE.  */
+static bool
+aarch64_simd_vector_alignment_reachable (const_tree type, bool is_packed)
+{
+  if (is_packed)
+    return false;
+
+  /* We guarantee alignment for vectors up to 128-bits.  */
+  if (tree_int_cst_compare (TYPE_SIZE (type),
+			    bitsize_int (BIGGEST_ALIGNMENT)) > 0)
+    return false;
+
+  /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned.  */
+  return true;
+}
+
 static unsigned HOST_WIDE_INT
 aarch64_shift_truncation_mask (enum machine_mode mode)
 {
@@ -6768,6 +6764,9 @@ aarch64_c_mode_for_suffix (char suffix)
 #undef TARGET_CLASS_MAX_NREGS
 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
 
+#undef TARGET_BUILTIN_DECL
+#define TARGET_BUILTIN_DECL aarch64_builtin_decl
+
 #undef  TARGET_EXPAND_BUILTIN
 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
 
@@ -6874,6 +6873,13 @@ aarch64_c_mode_for_suffix (char suffix)
 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
 
+#undef TARGET_VECTOR_ALIGNMENT
+#define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
+
+#undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
+#define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
+  aarch64_simd_vector_alignment_reachable
+
 struct gcc_target targetm = TARGET_INITIALIZER;
 
 #include "gt-aarch64.h"
diff --git a/gcc/config/aarch64/aarch64.h b/gcc/config/aarch64/aarch64.h
index d354bc2b6d1..8019f4c2a59 100644
--- a/gcc/config/aarch64/aarch64.h
+++ b/gcc/config/aarch64/aarch64.h
@@ -794,15 +794,6 @@ do {									     \
   extern void  __aarch64_sync_cache_range (void *, void *);	\
   __aarch64_sync_cache_range (beg, end)
 
-/* This should be integrated with the equivalent in the 32 bit
-   world.  */
-enum aarch64_builtins
-{
-  AARCH64_BUILTIN_MIN,
-  AARCH64_BUILTIN_THREAD_POINTER,
-  AARCH64_SIMD_BUILTIN_BASE
-};
-
 /*  VFP registers may only be accessed in the mode they
    were set.  */
 #define CANNOT_CHANGE_MODE_CLASS(FROM, TO, CLASS)	\
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index e41d64d3e73..7a1cdc89535 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -227,6 +227,7 @@
     UNSPEC_CMTST	; Used in aarch64-simd.md.
     UNSPEC_FMAX		; Used in aarch64-simd.md.
     UNSPEC_FMIN		; Used in aarch64-simd.md.
+    UNSPEC_BSL		; Used in aarch64-simd.md.
 ])
 
 ;; -------------------------------------------------------------------
diff --git a/gcc/testsuite/ChangeLog.aarch64 b/gcc/testsuite/ChangeLog.aarch64
index 2f1ad235d40..f765277c664 100644
--- a/gcc/testsuite/ChangeLog.aarch64
+++ b/gcc/testsuite/ChangeLog.aarch64
@@ -1,3 +1,9 @@
+2012-12-04  Marcus Shawcroft  <marcus.shawcroft@arm.com>
+
+	Backport form mainline
+	2012-12-04  Marcus Shawcroft  <marcus.shawcroft@arm.com>
+	* gcc.target/aarch64/121127.c: New test.
+
 2012-11-22  Ian Bolton  <ian.bolton@arm.com>
 
 	Backport from mainline
diff --git a/gcc/testsuite/gcc.target/aarch64/121127.c b/gcc/testsuite/gcc.target/aarch64/121127.c
new file mode 100644
index 00000000000..a7dca09fec8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/121127.c
@@ -0,0 +1,4 @@
+/* { dg-do compile } */
+/* { dg-options "-g -femit-struct-debug-baseonly" } */
+
+typedef __builtin_va_list __gnuc_va_list;
diff --git a/libgcc/ChangeLog.aarch64 b/libgcc/ChangeLog.aarch64
index 3f48587a728..53d718bf811 100644
--- a/libgcc/ChangeLog.aarch64
+++ b/libgcc/ChangeLog.aarch64
@@ -1,3 +1,9 @@
+2012-12-04  Marcus Shawcroft <marcus.shawcroft@arm.com>
+
+        * config/aarch64/sfp-machine.h (FP_EX_ALL): Define.
+	(FP_EX_SHIFT): Define.
+	(FP_TRAPPING_EXCEPTIONS): Define.
+
 2012-11-15  Marcus Shawcroft <marcus.shawcroft@arm.com>
 
 	* soft-fp: Updated from glibc upstream.
diff --git a/libgcc/config/aarch64/sfp-exceptions.c b/libgcc/config/aarch64/sfp-exceptions.c
new file mode 100644
index 00000000000..878cf8f95cd
--- /dev/null
+++ b/libgcc/config/aarch64/sfp-exceptions.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2012 Free Software Foundation, Inc.
+ *
+ * This file is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 3, or (at your option) any
+ * later version.
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * Under Section 7 of GPL version 3, you are granted additional
+ * permissions described in the GCC Runtime Library Exception, version
+ * 3.1, as published by the Free Software Foundation.
+ *
+ * You should have received a copy of the GNU General Public License and
+ * a copy of the GCC Runtime Library Exception along with this program;
+ * see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ */
+
+#include "sfp-machine.h"
+
+void
+__sfp_handle_exceptions (int _fex)
+{
+  const float fp_max = __FLT_MAX__;
+  const float fp_min = __FLT_MIN__;
+  const float fp_1e32 = 1.0e32f;
+  const float fp_zero = 0.0;
+  const float fp_one = 1.0;
+  unsigned fpsr;
+
+  if (_fex & FP_EX_INVALID)
+    {
+      __asm__ __volatile__ ("fdiv\ts0, %s0, %s0"
+			    :
+			    : "w" (fp_zero)
+			    : "s0");
+      __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));
+    }
+  if (_fex & FP_EX_DIVZERO)
+    {
+      __asm__ __volatile__ ("fdiv\ts0, %s0, %s1"
+			    :
+			    : "w" (fp_one), "w" (fp_zero)
+			    : "s0");
+      __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));
+    }
+  if (_fex & FP_EX_OVERFLOW)
+    {
+      __asm__ __volatile__ ("fadd\ts0, %s0, %s1"
+			    :
+			    : "w" (fp_max), "w" (fp_1e32)
+			    : "s0");
+      __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));
+    }
+  if (_fex & FP_EX_UNDERFLOW)
+    {
+      __asm__ __volatile__ ("fmul\ts0, %s0, %s0"
+			    :
+			    : "w" (fp_min)
+			    : "s0");
+      __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));
+    }
+  if (_fex & FP_EX_INEXACT)
+    {
+      __asm__ __volatile__ ("fsub\ts0, %s0, %s1"
+			    :
+			    : "w" (fp_max), "w" (fp_one)
+			    : "s0");
+      __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));
+    }
+}
diff --git a/libgcc/config/aarch64/sfp-machine.h b/libgcc/config/aarch64/sfp-machine.h
index 3a09ae7605f..52b6fb2c446 100644
--- a/libgcc/config/aarch64/sfp-machine.h
+++ b/libgcc/config/aarch64/sfp-machine.h
@@ -69,62 +69,26 @@ typedef int __gcc_CMPtype __attribute__ ((mode (__libgcc_cmp_return__)));
 #define FP_EX_OVERFLOW	0x04
 #define FP_EX_UNDERFLOW	0x08
 #define FP_EX_INEXACT	0x10
+#define FP_EX_SHIFT 8
+#define FP_EX_ALL \
+	(FP_EX_INVALID | FP_EX_DIVZERO | FP_EX_OVERFLOW | FP_EX_UNDERFLOW \
+	 | FP_EX_INEXACT)
 
-#define FP_HANDLE_EXCEPTIONS						\
-  do {									\
-    const float fp_max = __FLT_MAX__;					\
-    const float fp_min = __FLT_MIN__;					\
-    const float fp_1e32 = 1.0e32f;					\
-    const float fp_zero = 0.0;						\
-    const float fp_one = 1.0;						\
-    unsigned fpsr;							\
-    if (_fex & FP_EX_INVALID)						\
-      {									\
-        __asm__ __volatile__ ("fdiv\ts0, %s0, %s0"			\
-			      :						\
-			      : "w" (fp_zero)				\
-			      : "s0");					\
-	__asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));		\
-      }									\
-    if (_fex & FP_EX_DIVZERO)						\
-      {									\
-	__asm__ __volatile__ ("fdiv\ts0, %s0, %s1"			\
-			      :						\
-			      : "w" (fp_one), "w" (fp_zero)		\
-			      : "s0");					\
-	__asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));		\
-      }									\
-    if (_fex & FP_EX_OVERFLOW)						\
-      {									\
-        __asm__ __volatile__ ("fadd\ts0, %s0, %s1"			\
-			      :						\
-			      : "w" (fp_max), "w" (fp_1e32)		\
-			      : "s0");					\
-        __asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));		\
-      }									\
-    if (_fex & FP_EX_UNDERFLOW)						\
-      {									\
-	__asm__ __volatile__ ("fmul\ts0, %s0, %s0"			\
-			      :						\
-			      : "w" (fp_min)				\
-			      : "s0");					\
-	__asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));		\
-      }									\
-    if (_fex & FP_EX_INEXACT)						\
-      {									\
-	__asm__ __volatile__ ("fsub\ts0, %s0, %s1"			\
-			      :						\
-			      : "w" (fp_max), "w" (fp_one)		\
-			      : "s0");					\
-	__asm__ __volatile__ ("mrs\t%0, fpsr" : "=r" (fpsr));		\
-      }									\
-  } while (0)
+void __sfp_handle_exceptions (int);
 
+#define FP_HANDLE_EXCEPTIONS			\
+  do {						\
+    if (__builtin_expect (_fex, 0))		\
+      __sfp_handle_exceptions (_fex);		\
+  } while (0);
 
-#define FP_RND_NEAREST		0
-#define FP_RND_ZERO		0xc00000
+#define FP_TRAPPING_EXCEPTIONS ((_fpcr >> FP_EX_SHIFT) & FP_EX_ALL)
+
+#define FP_RND_NEAREST		0x000000
 #define FP_RND_PINF		0x400000
 #define FP_RND_MINF		0x800000
+#define FP_RND_ZERO		0xc00000
+#define FP_RND_MASK		0xc00000
 
 #define _FP_DECL_EX \
   unsigned long int _fpcr __attribute__ ((unused)) = FP_RND_NEAREST
@@ -135,7 +99,7 @@ typedef int __gcc_CMPtype __attribute__ ((mode (__libgcc_cmp_return__)));
 			  : "=r" (_fpcr));	\
   } while (0)
 
-#define FP_ROUNDMODE (_fpcr & 0xc00000)
+#define FP_ROUNDMODE (_fpcr & FP_RND_MASK)
 
 #define	__LITTLE_ENDIAN	1234
 #define	__BIG_ENDIAN	4321
diff --git a/libgcc/config/aarch64/t-softfp b/libgcc/config/aarch64/t-softfp
index 6500b5243e4..586dca22469 100644
--- a/libgcc/config/aarch64/t-softfp
+++ b/libgcc/config/aarch64/t-softfp
@@ -5,3 +5,5 @@ softfp_truncations := tfsf tfdf
 softfp_exclude_libgcc2 := n
 
 TARGET_LIBGCC2_CFLAGS += -Wno-missing-prototypes
+
+LIB2ADD += $(srcdir)/config/aarch64/sfp-exceptions.c