6919934: JSR 292 needs to support x86 C1

Summary: This implements JSR 292 support for C1 x86.
Reviewed-by: never, jrose, kvn

16 files changed, 183 insertions, 191 deletions

diff --git a/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp b/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp
index d5ea7dd43..6f4968200 100644
--- a/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_CodeStubs_sparc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -377,6 +377,16 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
 
 }
 
+
+void DeoptimizeStub::emit_code(LIR_Assembler* ce) {
+  __ bind(_entry);
+  __ call(SharedRuntime::deopt_blob()->unpack_with_reexecution());
+  __ delayed()->nop();
+  ce->add_call_info_here(_info);
+  debug_only(__ should_not_reach_here());
+}
+
+
 void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
   //---------------slow case: call to native-----------------
   __ bind(_entry);
diff --git a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
index 315e90019..4446addd9 100644
--- a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
@@ -378,12 +378,7 @@ int LIR_Assembler::emit_exception_handler() {
 
   int offset = code_offset();
 
-  if (compilation()->has_exception_handlers() || compilation()->env()->jvmti_can_post_on_exceptions()) {
-    __ call(Runtime1::entry_for(Runtime1::handle_exception_id), relocInfo::runtime_call_type);
-    __ delayed()->nop();
-  }
-
-  __ call(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type);
+  __ call(Runtime1::entry_for(Runtime1::handle_exception_id), relocInfo::runtime_call_type);
   __ delayed()->nop();
   debug_only(__ stop("should have gone to the caller");)
   assert(code_offset() - offset <= exception_handler_size, "overflow");
@@ -685,29 +680,29 @@ void LIR_Assembler::align_call(LIR_Code) {
 }
 
 
-void LIR_Assembler::call(address entry, relocInfo::relocType rtype, CodeEmitInfo* info) {
-  __ call(entry, rtype);
+void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
+  __ call(op->addr(), rtype);
   // the peephole pass fills the delay slot
 }
 
 
-void LIR_Assembler::ic_call(address entry, CodeEmitInfo* info) {
+void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
   RelocationHolder rspec = virtual_call_Relocation::spec(pc());
   __ set_oop((jobject)Universe::non_oop_word(), G5_inline_cache_reg);
   __ relocate(rspec);
-  __ call(entry, relocInfo::none);
+  __ call(op->addr(), relocInfo::none);
   // the peephole pass fills the delay slot
 }
 
 
-void LIR_Assembler::vtable_call(int vtable_offset, CodeEmitInfo* info) {
-  add_debug_info_for_null_check_here(info);
+void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
+  add_debug_info_for_null_check_here(op->info());
   __ ld_ptr(O0, oopDesc::klass_offset_in_bytes(), G3_scratch);
-  if (__ is_simm13(vtable_offset) ) {
-    __ ld_ptr(G3_scratch, vtable_offset, G5_method);
+  if (__ is_simm13(op->vtable_offset())) {
+    __ ld_ptr(G3_scratch, op->vtable_offset(), G5_method);
   } else {
     // This will generate 2 instructions
-    __ set(vtable_offset, G5_method);
+    __ set(op->vtable_offset(), G5_method);
     // ld_ptr, set_hi, set
     __ ld_ptr(G3_scratch, G5_method, G5_method);
   }
@@ -717,6 +712,16 @@ void LIR_Assembler::vtable_call(int vtable_offset, CodeEmitInfo* info) {
 }
 
 
+void LIR_Assembler::preserve_SP() {
+  Unimplemented();
+}
+
+
+void LIR_Assembler::restore_SP() {
+  Unimplemented();
+}
+
+
 // load with 32-bit displacement
 int LIR_Assembler::load(Register s, int disp, Register d, BasicType ld_type, CodeEmitInfo *info) {
   int load_offset = code_offset();
diff --git a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
index 094fae4a9..17423a9b9 100644
--- a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,17 +42,6 @@ void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
 }
 
 
-void C1_MacroAssembler::method_exit(bool restore_frame) {
-  // this code must be structured this way so that the return
-  // instruction can be a safepoint.
-  if (restore_frame) {
-    restore();
-  }
-  retl();
-  delayed()->nop();
-}
-
-
 void C1_MacroAssembler::explicit_null_check(Register base) {
   Unimplemented();
 }
diff --git a/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp b/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp
index 864d488ee..c88378f57 100644
--- a/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_Runtime1_sparc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -677,7 +677,7 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
         __ add(I7, frame::pc_return_offset, Oissuing_pc->after_save());
 
         __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address),
-                        Oissuing_pc->after_save());
+                        G2_thread, Oissuing_pc->after_save());
         __ verify_not_null_oop(Oexception->after_save());
         __ jmp(O0, 0);
         __ delayed()->restore();
@@ -985,7 +985,6 @@ OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 
 void Runtime1::generate_handle_exception(StubAssembler* sasm, OopMapSet* oop_maps, OopMap* oop_map, bool) {
   Label no_deopt;
-  Label no_handler;
 
   __ verify_not_null_oop(Oexception);
 
@@ -1003,9 +1002,14 @@ void Runtime1::generate_handle_exception(StubAssembler* sasm, OopMapSet* oop_map
   // whether it had a handler or not we will deoptimize
   // by entering the deopt blob with a pending exception.
 
+#ifdef ASSERT
+  Label done;
   __ tst(O0);
-  __ br(Assembler::zero, false, Assembler::pn, no_handler);
+  __ br(Assembler::notZero, false, Assembler::pn, done);
   __ delayed()->nop();
+  __ stop("should have found address");
+  __ bind(done);
+#endif
 
   // restore the registers that were saved at the beginning and jump to the exception handler.
   restore_live_registers(sasm);
@@ -1013,20 +1017,6 @@ void Runtime1::generate_handle_exception(StubAssembler* sasm, OopMapSet* oop_map
   __ jmp(O0, 0);
   __ delayed()->restore();
 
-  __ bind(no_handler);
-  __ mov(L0, I7); // restore return address
-
-  // restore exception oop
-  __ ld_ptr(G2_thread, in_bytes(JavaThread::exception_oop_offset()), Oexception->after_save());
-  __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_oop_offset()));
-
-  __ restore();
-
-  AddressLiteral exc(Runtime1::entry_for(Runtime1::unwind_exception_id));
-  __ jump_to(exc, G4);
-  __ delayed()->nop();
-
-
   oop_maps->add_gc_map(call_offset, oop_map);
 }
 
diff --git a/src/cpu/sparc/vm/interp_masm_sparc.cpp b/src/cpu/sparc/vm/interp_masm_sparc.cpp
index 604c2b9e4..9189f95d3 100644
--- a/src/cpu/sparc/vm/interp_masm_sparc.cpp
+++ b/src/cpu/sparc/vm/interp_masm_sparc.cpp
@@ -244,9 +244,10 @@ void InterpreterMacroAssembler::check_and_handle_earlyret(Register scratch_reg)
 }
 
 
-void InterpreterMacroAssembler::super_call_VM_leaf(Register thread_cache, address entry_point, Register arg_1) {
+void InterpreterMacroAssembler::super_call_VM_leaf(Register thread_cache, address entry_point, Register arg_1, Register arg_2) {
   mov(arg_1, O0);
-  MacroAssembler::call_VM_leaf_base(thread_cache, entry_point, 1);
+  mov(arg_2, O1);
+  MacroAssembler::call_VM_leaf_base(thread_cache, entry_point, 2);
 }
 #endif /* CC_INTERP */
 
diff --git a/src/cpu/sparc/vm/interp_masm_sparc.hpp b/src/cpu/sparc/vm/interp_masm_sparc.hpp
index 61d6a528d..cbb6fb4e2 100644
--- a/src/cpu/sparc/vm/interp_masm_sparc.hpp
+++ b/src/cpu/sparc/vm/interp_masm_sparc.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -121,7 +121,7 @@ class InterpreterMacroAssembler: public MacroAssembler {
                      bool check_exception = true);
 
 #ifndef CC_INTERP
-  void super_call_VM_leaf(Register thread_cache, address entry_point, Register arg_1);
+  void super_call_VM_leaf(Register thread_cache, address entry_point, Register arg_1, Register arg_2);
 
   // Generate a subtype check: branch to ok_is_subtype if sub_klass is
   // a subtype of super_klass.  Blows registers tmp1, tmp2 and tmp3.
diff --git a/src/cpu/sparc/vm/stubGenerator_sparc.cpp b/src/cpu/sparc/vm/stubGenerator_sparc.cpp
index 66c5a218d..091bc570d 100644
--- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp
+++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp
@@ -379,7 +379,7 @@ class StubGenerator: public StubCodeGenerator {
     __ save_frame(0);             // compensates for compiler weakness
     __ add(O7->after_save(), frame::pc_return_offset, Lscratch); // save the issuing PC
     BLOCK_COMMENT("call exception_handler_for_return_address");
-    __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), Lscratch);
+    __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), G2_thread, Lscratch);
     __ mov(O0, handler_reg);
     __ restore();                 // compensates for compiler weakness
 
diff --git a/src/cpu/sparc/vm/templateInterpreter_sparc.cpp b/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
index ada795d7e..8feef8bd8 100644
--- a/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
+++ b/src/cpu/sparc/vm/templateInterpreter_sparc.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1997-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1822,7 +1822,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
   __ add(issuing_pc_addr, Oissuing_pc->after_save());  // likewise set I1 to a value local to the caller
   __ super_call_VM_leaf(L7_thread_cache,
                         CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address),
-                        Oissuing_pc->after_save());
+                        G2_thread, Oissuing_pc->after_save());
 
   // The caller's SP was adjusted upon method entry to accomodate
   // the callee's non-argument locals. Undo that adjustment.
diff --git a/src/cpu/x86/vm/c1_CodeStubs_x86.cpp b/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
index c513092c5..139c95641 100644
--- a/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
+++ b/src/cpu/x86/vm/c1_CodeStubs_x86.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -373,6 +373,14 @@ void PatchingStub::emit_code(LIR_Assembler* ce) {
 }
 
 
+void DeoptimizeStub::emit_code(LIR_Assembler* ce) {
+  __ bind(_entry);
+  __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack_with_reexecution()));
+  ce->add_call_info_here(_info);
+  debug_only(__ should_not_reach_here());
+}
+
+
 void ImplicitNullCheckStub::emit_code(LIR_Assembler* ce) {
   ce->compilation()->implicit_exception_table()->append(_offset, __ offset());
   __ bind(_entry);
diff --git a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
index 85f370a4c..5907e058c 100644
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
+++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
@@ -436,40 +436,18 @@ int LIR_Assembler::emit_exception_handler() {
 
   int offset = code_offset();
 
-  // if the method does not have an exception handler, then there is
-  // no reason to search for one
-  if (compilation()->has_exception_handlers() || compilation()->env()->jvmti_can_post_on_exceptions()) {
-    // the exception oop and pc are in rax, and rdx
-    // no other registers need to be preserved, so invalidate them
-    __ invalidate_registers(false, true, true, false, true, true);
-
-    // check that there is really an exception
-    __ verify_not_null_oop(rax);
-
-    // search an exception handler (rax: exception oop, rdx: throwing pc)
-    __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_nofpu_id)));
-
-    // if the call returns here, then the exception handler for particular
-    // exception doesn't exist -> unwind activation and forward exception to caller
-  }
-
-  // the exception oop is in rax,
+  // the exception oop and pc are in rax, and rdx
   // no other registers need to be preserved, so invalidate them
-  __ invalidate_registers(false, true, true, true, true, true);
+  __ invalidate_registers(false, true, true, false, true, true);
 
   // check that there is really an exception
   __ verify_not_null_oop(rax);
 
-  // unlock the receiver/klass if necessary
-  // rax,: exception
-  ciMethod* method = compilation()->method();
-  if (method->is_synchronized() && GenerateSynchronizationCode) {
-    monitorexit(FrameMap::rbx_oop_opr, FrameMap::rcx_opr, SYNC_header, 0, rax);
-  }
+  // search an exception handler (rax: exception oop, rdx: throwing pc)
+  __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_nofpu_id)));
+
+  __ stop("should not reach here");
 
-  // unwind activation and forward exception to caller
-  // rax,: exception
-  __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
   assert(code_offset() - offset <= exception_handler_size, "overflow");
   __ end_a_stub();
 
@@ -495,8 +473,10 @@ int LIR_Assembler::emit_deopt_handler() {
 
   int offset = code_offset();
   InternalAddress here(__ pc());
+
   __ pushptr(here.addr());
   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
+
   assert(code_offset() - offset <= deopt_handler_size, "overflow");
   __ end_a_stub();
 
@@ -593,7 +573,7 @@ void LIR_Assembler::return_op(LIR_Opr result) {
   }
 
   // Pop the stack before the safepoint code
-  __ leave();
+  __ remove_frame(initial_frame_size_in_bytes());
 
   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 
@@ -2738,6 +2718,7 @@ void LIR_Assembler::align_call(LIR_Code code) {
     switch (code) {
       case lir_static_call:
       case lir_optvirtual_call:
+      case lir_dynamic_call:
         offset += NativeCall::displacement_offset;
         break;
       case lir_icvirtual_call:
@@ -2753,30 +2734,41 @@ void LIR_Assembler::align_call(LIR_Code code) {
 }
 
 
-void LIR_Assembler::call(address entry, relocInfo::relocType rtype, CodeEmitInfo* info) {
+void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
   assert(!os::is_MP() || (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
          "must be aligned");
-  __ call(AddressLiteral(entry, rtype));
-  add_call_info(code_offset(), info);
+  __ call(AddressLiteral(op->addr(), rtype));
+  add_call_info(code_offset(), op->info(), op->is_method_handle_invoke());
 }
 
 
-void LIR_Assembler::ic_call(address entry, CodeEmitInfo* info) {
+void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
   RelocationHolder rh = virtual_call_Relocation::spec(pc());
   __ movoop(IC_Klass, (jobject)Universe::non_oop_word());
   assert(!os::is_MP() ||
          (__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
          "must be aligned");
-  __ call(AddressLiteral(entry, rh));
-  add_call_info(code_offset(), info);
+  __ call(AddressLiteral(op->addr(), rh));
+  add_call_info(code_offset(), op->info(), op->is_method_handle_invoke());
 }
 
 
 /* Currently, vtable-dispatch is only enabled for sparc platforms */
-void LIR_Assembler::vtable_call(int vtable_offset, CodeEmitInfo* info) {
+void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
   ShouldNotReachHere();
 }
 
+
+void LIR_Assembler::preserve_SP() {
+  __ movptr(rbp, rsp);
+}
+
+
+void LIR_Assembler::restore_SP() {
+  __ movptr(rsp, rbp);
+}
+
+
 void LIR_Assembler::emit_static_call_stub() {
   address call_pc = __ pc();
   address stub = __ start_a_stub(call_stub_size);
@@ -2829,10 +2821,12 @@ void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmit
     } else {
       unwind_id = Runtime1::handle_exception_nofpu_id;
     }
+    __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
   } else {
-    unwind_id = Runtime1::unwind_exception_id;
+    // remove the activation
+    __ remove_frame(initial_frame_size_in_bytes());
+    __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
   }
-  __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
 
   // enough room for two byte trap
   __ nop();
diff --git a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
index c340c87c0..0dc5b173a 100644
--- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
+++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -317,14 +317,6 @@ void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
 }
 
 
-void C1_MacroAssembler::method_exit(bool restore_frame) {
-  if (restore_frame) {
-    leave();
-  }
-  ret(0);
-}
-
-
 void C1_MacroAssembler::build_frame(int frame_size_in_bytes) {
   // Make sure there is enough stack space for this method's activation.
   // Note that we do this before doing an enter(). This matches the
@@ -333,7 +325,7 @@ void C1_MacroAssembler::build_frame(int frame_size_in_bytes) {
   // between the two compilers.
   generate_stack_overflow_check(frame_size_in_bytes);
 
-  enter();
+  push(rbp);
 #ifdef TIERED
   // c2 leaves fpu stack dirty. Clean it on entry
   if (UseSSE < 2 ) {
@@ -344,6 +336,12 @@ void C1_MacroAssembler::build_frame(int frame_size_in_bytes) {
 }
 
 
+void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
+  increment(rsp, frame_size_in_bytes);  // Does not emit code for frame_size == 0
+  pop(rbp);
+}
+
+
 void C1_MacroAssembler::unverified_entry(Register receiver, Register ic_klass) {
   if (C1Breakpoint) int3();
   inline_cache_check(receiver, ic_klass);
diff --git a/src/cpu/x86/vm/c1_Runtime1_x86.cpp b/src/cpu/x86/vm/c1_Runtime1_x86.cpp
index 3a447754e..e3bf6fdae 100644
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright 1999-2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1999-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -688,18 +688,21 @@ void Runtime1::generate_handle_exception(StubAssembler *sasm, OopMapSet* oop_map
   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
   oop_maps->add_gc_map(call_offset, oop_map);
 
-  // rax,: handler address or NULL if no handler exists
+  // rax,: handler address
   //      will be the deopt blob if nmethod was deoptimized while we looked up
   //      handler regardless of whether handler existed in the nmethod.
 
   // only rax, is valid at this time, all other registers have been destroyed by the runtime call
   __ invalidate_registers(false, true, true, true, true, true);
 
+#ifdef ASSERT
   // Do we have an exception handler in the nmethod?
-  Label no_handler;
   Label done;
   __ testptr(rax, rax);
-  __ jcc(Assembler::zero, no_handler);
+  __ jcc(Assembler::notZero, done);
+  __ stop("no handler found");
+  __ bind(done);
+#endif
 
   // exception handler found
   // patch the return address -> the stub will directly return to the exception handler
@@ -712,36 +715,14 @@ void Runtime1::generate_handle_exception(StubAssembler *sasm, OopMapSet* oop_map
   __ leave();
   __ ret(0);
 
-  __ bind(no_handler);
-  // no exception handler found in this method, so the exception is
-  // forwarded to the caller (using the unwind code of the nmethod)
-  // there is no need to restore the registers
-
-  // restore the real return address that was saved before the RT-call
-  __ movptr(real_return_addr, Address(rsp, temp_1_off * VMRegImpl::stack_slot_size));
-  __ movptr(Address(rbp, 1*BytesPerWord), real_return_addr);
-
-  // load address of JavaThread object for thread-local data
-  NOT_LP64(__ get_thread(thread);)
-  // restore exception oop into rax, (convention for unwind code)
-  __ movptr(exception_oop, Address(thread, JavaThread::exception_oop_offset()));
-
-  // clear exception fields in JavaThread because they are no longer needed
-  // (fields must be cleared because they are processed by GC otherwise)
-  __ movptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
-  __ movptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
-
-  // pop the stub frame off
-  __ leave();
-
-  generate_unwind_exception(sasm);
-  __ stop("should not reach here");
 }
 
 
 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
   // incoming parameters
   const Register exception_oop = rax;
+  // callee-saved copy of exception_oop during runtime call
+  const Register exception_oop_callee_saved = NOT_LP64(rsi) LP64_ONLY(r14);
   // other registers used in this stub
   const Register exception_pc = rdx;
   const Register handler_addr = rbx;
@@ -769,38 +750,39 @@ void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
   // clear the FPU stack in case any FPU results are left behind
   __ empty_FPU_stack();
 
-  // leave activation of nmethod
-  __ leave();
-  // store return address (is on top of stack after leave)
-  __ movptr(exception_pc, Address(rsp, 0));
-
-  __ verify_oop(exception_oop);
+  // save exception_oop in callee-saved register to preserve it during runtime calls
+  __ verify_not_null_oop(exception_oop);
+  __ movptr(exception_oop_callee_saved, exception_oop);
 
-  // save exception oop from rax, to stack before call
-  __ push(exception_oop);
+  NOT_LP64(__ get_thread(thread);)
+  // Get return address (is on top of stack after leave).
+  __ movptr(exception_pc, Address(rsp, 0));
 
   // search the exception handler address of the caller (using the return address)
-  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), exception_pc);
-  // rax,: exception handler address of the caller
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
+  // rax: exception handler address of the caller
 
-  // only rax, is valid at this time, all other registers have been destroyed by the call
-  __ invalidate_registers(false, true, true, true, true, true);
+  // Only RAX and RSI are valid at this time, all other registers have been destroyed by the call.
+  __ invalidate_registers(false, true, true, true, false, true);
 
   // move result of call into correct register
   __ movptr(handler_addr, rax);
 
-  // restore exception oop in rax, (required convention of exception handler)
-  __ pop(exception_oop);
+  // Restore exception oop to RAX (required convention of exception handler).
+  __ movptr(exception_oop, exception_oop_callee_saved);
 
-  __ verify_oop(exception_oop);
+  // verify that there is really a valid exception in rax
+  __ verify_not_null_oop(exception_oop);
 
   // get throwing pc (= return address).
   // rdx has been destroyed by the call, so it must be set again
   // the pop is also necessary to simulate the effect of a ret(0)
   __ pop(exception_pc);
 
-  // verify that that there is really a valid exception in rax,
-  __ verify_not_null_oop(exception_oop);
+  // Restore SP from BP if the exception PC is a MethodHandle call site.
+  NOT_LP64(__ get_thread(thread);)
+  __ cmpl(Address(thread, JavaThread::is_method_handle_exception_offset()), 0);
+  __ cmovptr(Assembler::notEqual, rsp, rbp);
 
   // continue at exception handler (return address removed)
   // note: do *not* remove arguments when unwinding the
@@ -808,9 +790,9 @@ void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
   //       all arguments on the stack when entering the
   //       runtime to determine the exception handler
   //       (GC happens at call site with arguments!)
-  // rax,: exception oop
+  // rax: exception oop
   // rdx: throwing pc
-  // rbx,: exception handler
+  // rbx: exception handler
   __ jmp(handler_addr);
 }
 
diff --git a/src/cpu/x86/vm/stubGenerator_x86_32.cpp b/src/cpu/x86/vm/stubGenerator_x86_32.cpp
index 18f2fb7bc..e8276e518 100644
--- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp
+++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp
@@ -369,7 +369,7 @@ class StubGenerator: public StubCodeGenerator {
   // The pending exception in Thread is converted into a Java-level exception.
   //
   // Contract with Java-level exception handlers:
-  // rax,: exception
+  // rax: exception
   // rdx: throwing pc
   //
   // NOTE: At entry of this stub, exception-pc must be on stack !!
@@ -377,6 +377,12 @@ class StubGenerator: public StubCodeGenerator {
   address generate_forward_exception() {
     StubCodeMark mark(this, "StubRoutines", "forward exception");
     address start = __ pc();
+    const Register thread = rcx;
+
+    // other registers used in this stub
+    const Register exception_oop = rax;
+    const Register handler_addr  = rbx;
+    const Register exception_pc  = rdx;
 
     // Upon entry, the sp points to the return address returning into Java
     // (interpreted or compiled) code; i.e., the return address becomes the
@@ -389,8 +395,8 @@ class StubGenerator: public StubCodeGenerator {
 #ifdef ASSERT
     // make sure this code is only executed if there is a pending exception
     { Label L;
-      __ get_thread(rcx);
-      __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
+      __ get_thread(thread);
+      __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
       __ jcc(Assembler::notEqual, L);
       __ stop("StubRoutines::forward exception: no pending exception (1)");
       __ bind(L);
@@ -398,33 +404,40 @@ class StubGenerator: public StubCodeGenerator {
 #endif
 
     // compute exception handler into rbx,
-    __ movptr(rax, Address(rsp, 0));
+    __ get_thread(thread);
+    __ movptr(exception_pc, Address(rsp, 0));
     BLOCK_COMMENT("call exception_handler_for_return_address");
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rax);
-    __ mov(rbx, rax);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
+    __ mov(handler_addr, rax);
 
-    // setup rax, & rdx, remove return address & clear pending exception
-    __ get_thread(rcx);
-    __ pop(rdx);
-    __ movptr(rax, Address(rcx, Thread::pending_exception_offset()));
-    __ movptr(Address(rcx, Thread::pending_exception_offset()), NULL_WORD);
+    // setup rax & rdx, remove return address & clear pending exception
+    __ get_thread(thread);
+    __ pop(exception_pc);
+    __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
+    __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 
 #ifdef ASSERT
     // make sure exception is set
     { Label L;
-      __ testptr(rax, rax);
+      __ testptr(exception_oop, exception_oop);
       __ jcc(Assembler::notEqual, L);
       __ stop("StubRoutines::forward exception: no pending exception (2)");
       __ bind(L);
     }
 #endif
 
+    // Verify that there is really a valid exception in RAX.
+    __ verify_oop(exception_oop);
+
+    // Restore SP from BP if the exception PC is a MethodHandle call site.
+    __ cmpl(Address(thread, JavaThread::is_method_handle_exception_offset()), 0);
+    __ cmovptr(Assembler::notEqual, rsp, rbp);
+
     // continue at exception handler (return address removed)
-    // rax,: exception
-    // rbx,: exception handler
+    // rax: exception
+    // rbx: exception handler
     // rdx: throwing pc
-    __ verify_oop(rax);
-    __ jmp(rbx);
+    __ jmp(handler_addr);
 
     return start;
   }
diff --git a/src/cpu/x86/vm/stubGenerator_x86_64.cpp b/src/cpu/x86/vm/stubGenerator_x86_64.cpp
index 9415241f1..516a7826a 100644
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp
+++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp
@@ -466,7 +466,7 @@ class StubGenerator: public StubCodeGenerator {
     BLOCK_COMMENT("call exception_handler_for_return_address");
     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
                          SharedRuntime::exception_handler_for_return_address),
-                    c_rarg0);
+                    r15_thread, c_rarg0);
     __ mov(rbx, rax);
 
     // setup rax & rdx, remove return address & clear pending exception
diff --git a/src/cpu/x86/vm/templateInterpreter_x86_32.cpp b/src/cpu/x86/vm/templateInterpreter_x86_32.cpp
index eecfb3fd1..6174b2d8b 100644
--- a/src/cpu/x86/vm/templateInterpreter_x86_32.cpp
+++ b/src/cpu/x86/vm/templateInterpreter_x86_32.cpp
@@ -1550,6 +1550,7 @@ int AbstractInterpreter::layout_activation(methodOop method,
 void TemplateInterpreterGenerator::generate_throw_exception() {
   // Entry point in previous activation (i.e., if the caller was interpreted)
   Interpreter::_rethrow_exception_entry = __ pc();
+  const Register thread = rcx;
 
   // Restore sp to interpreter_frame_last_sp even though we are going
   // to empty the expression stack for the exception processing.
@@ -1598,10 +1599,10 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
   // Set the popframe_processing bit in pending_popframe_condition indicating that we are
   // currently handling popframe, so that call_VMs that may happen later do not trigger new
   // popframe handling cycles.
-  __ get_thread(rcx);
-  __ movl(rdx, Address(rcx, JavaThread::popframe_condition_offset()));
+  __ get_thread(thread);
+  __ movl(rdx, Address(thread, JavaThread::popframe_condition_offset()));
   __ orl(rdx, JavaThread::popframe_processing_bit);
-  __ movl(Address(rcx, JavaThread::popframe_condition_offset()), rdx);
+  __ movl(Address(thread, JavaThread::popframe_condition_offset()), rdx);
 
   {
     // Check to see whether we are returning to a deoptimized frame.
@@ -1629,8 +1630,8 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
     __ subptr(rdi, rax);
     __ addptr(rdi, wordSize);
     // Save these arguments
-    __ get_thread(rcx);
-    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), rcx, rax, rdi);
+    __ get_thread(thread);
+    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), thread, rax, rdi);
 
     __ remove_activation(vtos, rdx,
                          /* throw_monitor_exception */ false,
@@ -1638,8 +1639,8 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
                          /* notify_jvmdi */ false);
 
     // Inform deoptimization that it is responsible for restoring these arguments
-    __ get_thread(rcx);
-    __ movl(Address(rcx, JavaThread::popframe_condition_offset()), JavaThread::popframe_force_deopt_reexecution_bit);
+    __ get_thread(thread);
+    __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_force_deopt_reexecution_bit);
 
     // Continue in deoptimization handler
     __ jmp(rdx);
@@ -1665,12 +1666,12 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
   // expression stack if necessary.
   __ mov(rax, rsp);
   __ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
-  __ get_thread(rcx);
+  __ get_thread(thread);
   // PC must point into interpreter here
-  __ set_last_Java_frame(rcx, noreg, rbp, __ pc());
-  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), rcx, rax, rbx);
-  __ get_thread(rcx);
-  __ reset_last_Java_frame(rcx, true, true);
+  __ set_last_Java_frame(thread, noreg, rbp, __ pc());
+  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
+  __ get_thread(thread);
+  __ reset_last_Java_frame(thread, true, true);
   // Restore the last_sp and null it out
   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
@@ -1684,8 +1685,8 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
   }
 
   // Clear the popframe condition flag
-  __ get_thread(rcx);
-  __ movl(Address(rcx, JavaThread::popframe_condition_offset()), JavaThread::popframe_inactive);
+  __ get_thread(thread);
+  __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_inactive);
 
   __ dispatch_next(vtos);
   // end of PopFrame support
@@ -1694,27 +1695,27 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
 
   // preserve exception over this code sequence
   __ pop_ptr(rax);
-  __ get_thread(rcx);
-  __ movptr(Address(rcx, JavaThread::vm_result_offset()), rax);
+  __ get_thread(thread);
+  __ movptr(Address(thread, JavaThread::vm_result_offset()), rax);
   // remove the activation (without doing throws on illegalMonitorExceptions)
   __ remove_activation(vtos, rdx, false, true, false);
   // restore exception
-  __ get_thread(rcx);
-  __ movptr(rax, Address(rcx, JavaThread::vm_result_offset()));
-  __ movptr(Address(rcx, JavaThread::vm_result_offset()), NULL_WORD);
+  __ get_thread(thread);
+  __ movptr(rax, Address(thread, JavaThread::vm_result_offset()));
+  __ movptr(Address(thread, JavaThread::vm_result_offset()), NULL_WORD);
   __ verify_oop(rax);
 
   // Inbetween activations - previous activation type unknown yet
   // compute continuation point - the continuation point expects
   // the following registers set up:
   //
-  // rax,: exception
+  // rax: exception
   // rdx: return address/pc that threw exception
   // rsp: expression stack of caller
-  // rbp,: rbp, of caller
+  // rbp: rbp, of caller
   __ push(rax);                                  // save exception
   __ push(rdx);                                  // save return address
-  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rdx);
+  __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, rdx);
   __ mov(rbx, rax);                              // save exception handler
   __ pop(rdx);                                   // restore return address
   __ pop(rax);                                   // restore exception
@@ -1728,6 +1729,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
 //
 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
   address entry = __ pc();
+  const Register thread = rcx;
 
   __ restore_bcp();
   __ restore_locals();
@@ -1735,8 +1737,8 @@ address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state
   __ empty_FPU_stack();
   __ load_earlyret_value(state);
 
-  __ get_thread(rcx);
-  __ movptr(rcx, Address(rcx, JavaThread::jvmti_thread_state_offset()));
+  __ get_thread(thread);
+  __ movptr(rcx, Address(thread, JavaThread::jvmti_thread_state_offset()));
   const Address cond_addr(rcx, JvmtiThreadState::earlyret_state_offset());
 
   // Clear the earlyret state
diff --git a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
index 442254416..da5a0eca9 100644
--- a/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
+++ b/src/cpu/x86/vm/templateInterpreter_x86_64.cpp
@@ -1741,7 +1741,7 @@ void TemplateInterpreterGenerator::generate_throw_exception() {
   __ push(rdx);                                  // save return address
   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
                           SharedRuntime::exception_handler_for_return_address),
-                        rdx);
+                        r15_thread, rdx);
   __ mov(rbx, rax);                              // save exception handler
   __ pop(rdx);                                   // restore return address
   __ pop(rax);                                   // restore exception