Merge

21 files changed, 572 insertions, 126 deletions

diff --git a/src/cpu/sparc/vm/assembler_sparc.cpp b/src/cpu/sparc/vm/assembler_sparc.cpp
index c4e319d0a..25c850c8b 100644
--- a/src/cpu/sparc/vm/assembler_sparc.cpp
+++ b/src/cpu/sparc/vm/assembler_sparc.cpp
@@ -1641,6 +1641,21 @@ void  MacroAssembler::set_narrow_oop(jobject obj, Register d) {
 
 }
 
+void  MacroAssembler::set_narrow_klass(Klass* k, Register d) {
+  assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int klass_index = oop_recorder()->find_index(k);
+  RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+  narrowOop encoded_k = oopDesc::encode_klass(k);
+
+  assert_not_delayed();
+  // Relocation with special format (see relocInfo_sparc.hpp).
+  relocate(rspec, 1);
+  // Assembler::sethi(encoded_k, d);
+  emit_long( op(branch_op) | rd(d) | op2(sethi_op2) | hi22(encoded_k) );
+  // Don't add relocation for 'add'. Do patching during 'sethi' processing.
+  add(d, low10(encoded_k), d);
+
+}
 
 void MacroAssembler::align(int modulus) {
   while (offset() % modulus != 0) nop();
@@ -4660,7 +4675,7 @@ void MacroAssembler::load_klass(Register src_oop, Register klass) {
   // if this changes, change that.
   if (UseCompressedKlassPointers) {
     lduw(src_oop, oopDesc::klass_offset_in_bytes(), klass);
-    decode_heap_oop_not_null(klass);
+    decode_klass_not_null(klass);
   } else {
     ld_ptr(src_oop, oopDesc::klass_offset_in_bytes(), klass);
   }
@@ -4669,7 +4684,7 @@ void MacroAssembler::load_klass(Register src_oop, Register klass) {
 void MacroAssembler::store_klass(Register klass, Register dst_oop) {
   if (UseCompressedKlassPointers) {
     assert(dst_oop != klass, "not enough registers");
-    encode_heap_oop_not_null(klass);
+    encode_klass_not_null(klass);
     st(klass, dst_oop, oopDesc::klass_offset_in_bytes());
   } else {
     st_ptr(klass, dst_oop, oopDesc::klass_offset_in_bytes());
@@ -4829,17 +4844,58 @@ void  MacroAssembler::decode_heap_oop_not_null(Register src, Register dst) {
   // pd_code_size_limit.
   // Also do not verify_oop as this is called by verify_oop.
   assert (UseCompressedOops, "must be compressed");
-  assert (Universe::heap() != NULL, "java heap should be initialized");
   assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
   sllx(src, LogMinObjAlignmentInBytes, dst);
   if (Universe::narrow_oop_base() != NULL)
     add(dst, G6_heapbase, dst);
 }
 
+void MacroAssembler::encode_klass_not_null(Register r) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  assert (UseCompressedKlassPointers, "must be compressed");
+  assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+  if (Universe::narrow_klass_base() != NULL)
+    sub(r, G6_heapbase, r);
+  srlx(r, LogKlassAlignmentInBytes, r);
+}
+
+void MacroAssembler::encode_klass_not_null(Register src, Register dst) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  assert (UseCompressedKlassPointers, "must be compressed");
+  assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+  if (Universe::narrow_klass_base() == NULL) {
+    srlx(src, LogKlassAlignmentInBytes, dst);
+  } else {
+    sub(src, G6_heapbase, dst);
+    srlx(dst, LogKlassAlignmentInBytes, dst);
+  }
+}
+
+void  MacroAssembler::decode_klass_not_null(Register r) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  // Do not add assert code to this unless you change vtableStubs_sparc.cpp
+  // pd_code_size_limit.
+  assert (UseCompressedKlassPointers, "must be compressed");
+  assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+  sllx(r, LogKlassAlignmentInBytes, r);
+  if (Universe::narrow_klass_base() != NULL)
+    add(r, G6_heapbase, r);
+}
+
+void  MacroAssembler::decode_klass_not_null(Register src, Register dst) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  // Do not add assert code to this unless you change vtableStubs_sparc.cpp
+  // pd_code_size_limit.
+  assert (UseCompressedKlassPointers, "must be compressed");
+  assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+  sllx(src, LogKlassAlignmentInBytes, dst);
+  if (Universe::narrow_klass_base() != NULL)
+    add(dst, G6_heapbase, dst);
+}
+
 void MacroAssembler::reinit_heapbase() {
-  if (UseCompressedOops) {
-    // call indirectly to solve generation ordering problem
-    AddressLiteral base(Universe::narrow_oop_base_addr());
+  if (UseCompressedOops || UseCompressedKlassPointers) {
+    AddressLiteral base(Universe::narrow_ptrs_base_addr());
     load_ptr_contents(base, G6_heapbase);
   }
 }
diff --git a/src/cpu/sparc/vm/assembler_sparc.hpp b/src/cpu/sparc/vm/assembler_sparc.hpp
index 48f76a85d..f9b893258 100644
--- a/src/cpu/sparc/vm/assembler_sparc.hpp
+++ b/src/cpu/sparc/vm/assembler_sparc.hpp
@@ -2280,6 +2280,11 @@ public:
   void encode_heap_oop_not_null(Register src, Register dst);
   void decode_heap_oop_not_null(Register src, Register dst);
 
+  void encode_klass_not_null(Register r);
+  void decode_klass_not_null(Register r);
+  void encode_klass_not_null(Register src, Register dst);
+  void decode_klass_not_null(Register src, Register dst);
+
   // Support for managing the JavaThread pointer (i.e.; the reference to
   // thread-local information).
   void get_thread();                                // load G2_thread
@@ -2409,6 +2414,7 @@ public:
   inline void    set_metadata             (const AddressLiteral& obj_addr, Register d); // same as load_address
 
   void set_narrow_oop( jobject obj, Register d );
+  void set_narrow_klass( Klass* k, Register d );
 
   // nop padding
   void align(int modulus);
diff --git a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
index 3f26a2b57..27bf44244 100644
--- a/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_LIRAssembler_sparc.cpp
@@ -105,6 +105,11 @@ bool LIR_Assembler::is_single_instruction(LIR_Op* op) {
         if (src->is_address() && !src->is_stack() && (src->type() == T_OBJECT || src->type() == T_ARRAY)) return false;
       }
 
+      if (UseCompressedKlassPointers) {
+        if (src->is_address() && !src->is_stack() && src->type() == T_ADDRESS &&
+            src->as_address_ptr()->disp() == oopDesc::klass_offset_in_bytes()) return false;
+      }
+
       if (dst->is_register()) {
         if (src->is_address() && Assembler::is_simm13(src->as_address_ptr()->disp())) {
           return !PatchALot;
@@ -969,8 +974,18 @@ int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType typ
 #endif
         }
         break;
-      case T_METADATA:
-      case T_ADDRESS:  __ ld_ptr(base, offset, to_reg->as_register()); break;
+      case T_METADATA:  __ ld_ptr(base, offset, to_reg->as_register()); break;
+      case T_ADDRESS:
+#ifdef _LP64
+        if (offset == oopDesc::klass_offset_in_bytes() && UseCompressedKlassPointers) {
+          __ lduw(base, offset, to_reg->as_register());
+          __ decode_klass_not_null(to_reg->as_register());
+        } else
+#endif
+        {
+          __ ld_ptr(base, offset, to_reg->as_register());
+        }
+        break;
       case T_ARRAY : // fall through
       case T_OBJECT:
         {
@@ -2344,7 +2359,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     if (UseCompressedKlassPointers) {
       // tmp holds the default type. It currently comes uncompressed after the
       // load of a constant, so encode it.
-      __ encode_heap_oop(tmp);
+      __ encode_klass_not_null(tmp);
       // load the raw value of the dst klass, since we will be comparing
       // uncompressed values directly.
       __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
diff --git a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
index be56a7031..05db22c93 100644
--- a/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
+++ b/src/cpu/sparc/vm/c1_MacroAssembler_sparc.cpp
@@ -189,7 +189,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
   if (UseCompressedKlassPointers) {
     // Save klass
     mov(klass, t1);
-    encode_heap_oop_not_null(t1);
+    encode_klass_not_null(t1);
     stw(t1, obj, oopDesc::klass_offset_in_bytes());
   } else {
     st_ptr(klass, obj, oopDesc::klass_offset_in_bytes());
diff --git a/src/cpu/sparc/vm/methodHandles_sparc.cpp b/src/cpu/sparc/vm/methodHandles_sparc.cpp
index f8b5f24d5..9245f3b40 100644
--- a/src/cpu/sparc/vm/methodHandles_sparc.cpp
+++ b/src/cpu/sparc/vm/methodHandles_sparc.cpp
@@ -208,8 +208,6 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
   Register O1_scratch    = O1;
   Register O4_param_size = O4;   // size of parameters
 
-  address code_start = __ pc();
-
   // here's where control starts out:
   __ align(CodeEntryAlignment);
   address entry_point = __ pc();
@@ -252,22 +250,9 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
   // O4_first_arg_addr is live!
 
   if (TraceMethodHandles) {
-    const char* name = vmIntrinsics::name_at(iid);
-    if (*name == '_')  name += 1;
-    const size_t len = strlen(name) + 50;
-    char* qname = NEW_C_HEAP_ARRAY(char, len, mtInternal);
-    const char* suffix = "";
-    if (vmIntrinsics::method_for(iid) == NULL ||
-        !vmIntrinsics::method_for(iid)->access_flags().is_public()) {
-      if (is_signature_polymorphic_static(iid))
-        suffix = "/static";
-      else
-        suffix = "/private";
-    }
-    jio_snprintf(qname, len, "MethodHandle::interpreter_entry::%s%s", name, suffix);
     if (O0_mh != noreg)
       __ mov(O0_mh, G3_method_handle);  // make stub happy
-    trace_method_handle(_masm, qname);
+    trace_method_handle_interpreter_entry(_masm, iid);
   }
 
   if (iid == vmIntrinsics::_invokeBasic) {
@@ -287,14 +272,6 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
     generate_method_handle_dispatch(_masm, iid, O0_recv, G5_member, not_for_compiler_entry);
   }
 
-  if (PrintMethodHandleStubs) {
-    address code_end = __ pc();
-    tty->print_cr("--------");
-    tty->print_cr("method handle interpreter entry for %s", vmIntrinsics::name_at(iid));
-    Disassembler::decode(code_start, code_end);
-    tty->cr();
-  }
-
   return entry_point;
 }
 
diff --git a/src/cpu/sparc/vm/methodHandles_sparc.hpp b/src/cpu/sparc/vm/methodHandles_sparc.hpp
index 75231984d..d58e7d75c 100644
--- a/src/cpu/sparc/vm/methodHandles_sparc.hpp
+++ b/src/cpu/sparc/vm/methodHandles_sparc.hpp
@@ -58,5 +58,3 @@ public:
                                   Register recv, Register method_temp,
                                   Register temp2, Register temp3,
                                   bool for_compiler_entry);
-
-  static void trace_method_handle(MacroAssembler* _masm, const char* adaptername) PRODUCT_RETURN;
diff --git a/src/cpu/sparc/vm/relocInfo_sparc.cpp b/src/cpu/sparc/vm/relocInfo_sparc.cpp
index fcaaa18a6..aeb2420c6 100644
--- a/src/cpu/sparc/vm/relocInfo_sparc.cpp
+++ b/src/cpu/sparc/vm/relocInfo_sparc.cpp
@@ -97,8 +97,8 @@ void Relocation::pd_set_data_value(address x, intptr_t o, bool verify_only) {
     jint inst2;
     guarantee(Assembler::inv_op2(inst)==Assembler::sethi_op2, "must be sethi");
     if (format() != 0) {
-      assert(type() == relocInfo::oop_type, "only narrow oops case");
-      jint np = oopDesc::encode_heap_oop((oop)x);
+      assert(type() == relocInfo::oop_type || type() == relocInfo::metadata_type, "only narrow oops or klasses case");
+      jint np = type() == relocInfo::oop_type ? oopDesc::encode_heap_oop((oop)x) : oopDesc::encode_klass((Klass*)x);
       inst &= ~Assembler::hi22(-1);
       inst |=  Assembler::hi22((intptr_t)np);
       if (verify_only) {
diff --git a/src/cpu/sparc/vm/sparc.ad b/src/cpu/sparc/vm/sparc.ad
index f6e7d03b6..b7029faa6 100644
--- a/src/cpu/sparc/vm/sparc.ad
+++ b/src/cpu/sparc/vm/sparc.ad
@@ -557,9 +557,9 @@ int MachCallDynamicJavaNode::ret_addr_offset() {
     int entry_offset = InstanceKlass::vtable_start_offset() + vtable_index*vtableEntry::size();
     int v_off = entry_offset*wordSize + vtableEntry::method_offset_in_bytes();
     int klass_load_size;
-    if (UseCompressedOops && UseCompressedKlassPointers) {
+    if (UseCompressedKlassPointers) {
       assert(Universe::heap() != NULL, "java heap should be initialized");
-      if (Universe::narrow_oop_base() == NULL)
+      if (Universe::narrow_klass_base() == NULL)
         klass_load_size = 2*BytesPerInstWord; // see MacroAssembler::load_klass()
       else
         klass_load_size = 3*BytesPerInstWord;
@@ -1707,11 +1707,11 @@ uint reloc_java_to_interp() {
 void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
   st->print_cr("\nUEP:");
 #ifdef    _LP64
-  if (UseCompressedOops) {
+  if (UseCompressedKlassPointers) {
     assert(Universe::heap() != NULL, "java heap should be initialized");
     st->print_cr("\tLDUW   [R_O0 + oopDesc::klass_offset_in_bytes],R_G5\t! Inline cache check - compressed klass");
     st->print_cr("\tSLL    R_G5,3,R_G5");
-    if (Universe::narrow_oop_base() != NULL)
+    if (Universe::narrow_klass_base() != NULL)
       st->print_cr("\tADD    R_G5,R_G6_heap_base,R_G5");
   } else {
     st->print_cr("\tLDX    [R_O0 + oopDesc::klass_offset_in_bytes],R_G5\t! Inline cache check");
@@ -1942,6 +1942,12 @@ bool Matcher::narrow_oop_use_complex_address() {
   return false;
 }
 
+bool Matcher::narrow_klass_use_complex_address() {
+  NOT_LP64(ShouldNotCallThis());
+  assert(UseCompressedKlassPointers, "only for compressed klass code");
+  return false;
+}
+
 // Is it better to copy float constants, or load them directly from memory?
 // Intel can load a float constant from a direct address, requiring no
 // extra registers.  Most RISCs will have to materialize an address into a
@@ -2602,9 +2608,9 @@ encode %{
       int off = __ offset();
       __ load_klass(O0, G3_scratch);
       int klass_load_size;
-      if (UseCompressedOops && UseCompressedKlassPointers) {
+      if (UseCompressedKlassPointers) {
         assert(Universe::heap() != NULL, "java heap should be initialized");
-        if (Universe::narrow_oop_base() == NULL)
+        if (Universe::narrow_klass_base() == NULL)
           klass_load_size = 2*BytesPerInstWord;
         else
           klass_load_size = 3*BytesPerInstWord;
@@ -3610,6 +3616,15 @@ operand immN()
   interface(CONST_INTER);
 %}
 
+operand immNKlass()
+%{
+  match(ConNKlass);
+
+  op_cost(10);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // NULL Pointer Immediate
 operand immN0()
 %{
@@ -5870,8 +5885,8 @@ instruct loadI2L_immI(iRegL dst, memory mem, immI mask, iRegL tmp) %{
 %}
 
 // Load Unsigned Integer into a Long Register
-instruct loadUI2L(iRegL dst, memory mem) %{
-  match(Set dst (LoadUI2L mem));
+instruct loadUI2L(iRegL dst, memory mem, immL_32bits mask) %{
+  match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
   ins_cost(MEMORY_REF_COST);
 
   size(4);
@@ -6159,6 +6174,17 @@ instruct loadConN(iRegN dst, immN src) %{
   ins_pipe(ialu_hi_lo_reg);
 %}
 
+instruct loadConNKlass(iRegN dst, immNKlass src) %{
+  match(Set dst src);
+  ins_cost(DEFAULT_COST * 3/2);
+  format %{ "SET    $src,$dst\t! compressed klass ptr" %}
+  ins_encode %{
+    Register dst = $dst$$Register;
+    __ set_narrow_klass((Klass*)$src$$constant, dst);
+  %}
+  ins_pipe(ialu_hi_lo_reg);
+%}
+
 // Materialize long value (predicated by immL_cheap).
 instruct loadConL_set64(iRegL dst, immL_cheap con, o7RegL tmp) %{
   match(Set dst con);
@@ -6475,6 +6501,25 @@ instruct storeN(memory dst, iRegN src) %{
    ins_pipe(istore_mem_spORreg);
 %}
 
+instruct storeNKlass(memory dst, iRegN src) %{
+   match(Set dst (StoreNKlass dst src));
+   ins_cost(MEMORY_REF_COST);
+   size(4);
+
+   format %{ "STW    $src,$dst\t! compressed klass ptr" %}
+   ins_encode %{
+     Register base = as_Register($dst$$base);
+     Register index = as_Register($dst$$index);
+     Register src = $src$$Register;
+     if (index != G0) {
+       __ stw(src, base, index);
+     } else {
+       __ stw(src, base, $dst$$disp);
+     }
+   %}
+   ins_pipe(istore_mem_spORreg);
+%}
+
 instruct storeN0(memory dst, immN0 src) %{
    match(Set dst (StoreN dst src));
    ins_cost(MEMORY_REF_COST);
@@ -6582,6 +6627,23 @@ instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
   ins_pipe(ialu_reg);
 %}
 
+instruct encodeKlass_not_null(iRegN dst, iRegP src) %{
+  match(Set dst (EncodePKlass src));
+  format %{ "encode_klass_not_null $src, $dst" %}
+  ins_encode %{
+    __ encode_klass_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
+
+instruct decodeKlass_not_null(iRegP dst, iRegN src) %{
+  match(Set dst (DecodeNKlass src));
+  format %{ "decode_klass_not_null $src, $dst" %}
+  ins_encode %{
+    __ decode_klass_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
 
 //----------MemBar Instructions-----------------------------------------------
 // Memory barrier flavors
diff --git a/src/cpu/sparc/vm/vm_version_sparc.cpp b/src/cpu/sparc/vm/vm_version_sparc.cpp
index 07eadd5bf..aaa601fbb 100644
--- a/src/cpu/sparc/vm/vm_version_sparc.cpp
+++ b/src/cpu/sparc/vm/vm_version_sparc.cpp
@@ -117,6 +117,7 @@ void VM_Version::initialize() {
     // 32-bit oops don't make sense for the 64-bit VM on sparc
     // since the 32-bit VM has the same registers and smaller objects.
     Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes);
+    Universe::set_narrow_klass_shift(LogKlassAlignmentInBytes);
 #endif // _LP64
 #ifdef COMPILER2
     // Indirect branch is the same cost as direct
diff --git a/src/cpu/sparc/vm/vtableStubs_sparc.cpp b/src/cpu/sparc/vm/vtableStubs_sparc.cpp
index 8a1f6ec5e..8e7021fa1 100644
--- a/src/cpu/sparc/vm/vtableStubs_sparc.cpp
+++ b/src/cpu/sparc/vm/vtableStubs_sparc.cpp
@@ -220,13 +220,13 @@ int VtableStub::pd_code_size_limit(bool is_vtable_stub) {
       const int basic = 5*BytesPerInstWord +
                         // shift;add for load_klass (only shift with zero heap based)
                         (UseCompressedKlassPointers ?
-                         ((Universe::narrow_oop_base() == NULL) ? BytesPerInstWord : 2*BytesPerInstWord) : 0);
+                         ((Universe::narrow_klass_base() == NULL) ? BytesPerInstWord : 2*BytesPerInstWord) : 0);
       return basic + slop;
     } else {
       const int basic = (28 LP64_ONLY(+ 6)) * BytesPerInstWord +
                         // shift;add for load_klass (only shift with zero heap based)
                         (UseCompressedKlassPointers ?
-                         ((Universe::narrow_oop_base() == NULL) ? BytesPerInstWord : 2*BytesPerInstWord) : 0);
+                         ((Universe::narrow_klass_base() == NULL) ? BytesPerInstWord : 2*BytesPerInstWord) : 0);
       return (basic + slop);
     }
   }
diff --git a/src/cpu/x86/vm/assembler_x86.cpp b/src/cpu/x86/vm/assembler_x86.cpp
index 734d739a6..378c1f8c6 100644
--- a/src/cpu/x86/vm/assembler_x86.cpp
+++ b/src/cpu/x86/vm/assembler_x86.cpp
@@ -6916,7 +6916,7 @@ void MacroAssembler::call_VM_base(Register oop_result,
 #ifdef ASSERT
   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
   // r12 is the heapbase.
-  LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base");)
+  LP64_ONLY(if ((UseCompressedOops || UseCompressedKlassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 #endif // ASSERT
 
   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
@@ -10016,7 +10016,7 @@ void MacroAssembler::load_klass(Register dst, Register src) {
 #ifdef _LP64
   if (UseCompressedKlassPointers) {
     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
-    decode_heap_oop_not_null(dst);
+    decode_klass_not_null(dst);
   } else
 #endif
     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
@@ -10027,15 +10027,10 @@ void MacroAssembler::load_prototype_header(Register dst, Register src) {
   if (UseCompressedKlassPointers) {
     assert (Universe::heap() != NULL, "java heap should be initialized");
     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
-    if (Universe::narrow_oop_shift() != 0) {
-      assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
-      if (LogMinObjAlignmentInBytes == Address::times_8) {
-        movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset()));
-      } else {
-        // OK to use shift since we don't need to preserve flags.
-        shlq(dst, LogMinObjAlignmentInBytes);
-        movq(dst, Address(r12_heapbase, dst, Address::times_1, Klass::prototype_header_offset()));
-      }
+    if (Universe::narrow_klass_shift() != 0) {
+      assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+      assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
+      movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset()));
     } else {
       movq(dst, Address(dst, Klass::prototype_header_offset()));
     }
@@ -10050,7 +10045,7 @@ void MacroAssembler::load_prototype_header(Register dst, Register src) {
 void MacroAssembler::store_klass(Register dst, Register src) {
 #ifdef _LP64
   if (UseCompressedKlassPointers) {
-    encode_heap_oop_not_null(src);
+    encode_klass_not_null(src);
     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
   } else
 #endif
@@ -10132,12 +10127,12 @@ void MacroAssembler::store_klass_gap(Register dst, Register src) {
 
 #ifdef ASSERT
 void MacroAssembler::verify_heapbase(const char* msg) {
-  assert (UseCompressedOops, "should be compressed");
+  assert (UseCompressedOops || UseCompressedKlassPointers, "should be compressed");
   assert (Universe::heap() != NULL, "java heap should be initialized");
   if (CheckCompressedOops) {
     Label ok;
     push(rscratch1); // cmpptr trashes rscratch1
-    cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
+    cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
     jcc(Assembler::equal, ok);
     STOP(msg);
     bind(ok);
@@ -10275,6 +10270,74 @@ void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
   }
 }
 
+void MacroAssembler::encode_klass_not_null(Register r) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+#ifdef ASSERT
+  verify_heapbase("MacroAssembler::encode_klass_not_null: heap base corrupted?");
+#endif
+  if (Universe::narrow_klass_base() != NULL) {
+    subq(r, r12_heapbase);
+  }
+  if (Universe::narrow_klass_shift() != 0) {
+    assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+    shrq(r, LogKlassAlignmentInBytes);
+  }
+}
+
+void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+#ifdef ASSERT
+  verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
+#endif
+  if (dst != src) {
+    movq(dst, src);
+  }
+  if (Universe::narrow_klass_base() != NULL) {
+    subq(dst, r12_heapbase);
+  }
+  if (Universe::narrow_klass_shift() != 0) {
+    assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+    shrq(dst, LogKlassAlignmentInBytes);
+  }
+}
+
+void  MacroAssembler::decode_klass_not_null(Register r) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  // Note: it will change flags
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  // Cannot assert, unverified entry point counts instructions (see .ad file)
+  // vtableStubs also counts instructions in pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
+  if (Universe::narrow_klass_shift() != 0) {
+    assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+    shlq(r, LogKlassAlignmentInBytes);
+    if (Universe::narrow_klass_base() != NULL) {
+      addq(r, r12_heapbase);
+    }
+  } else {
+    assert (Universe::narrow_klass_base() == NULL, "sanity");
+  }
+}
+
+void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
+  assert(Metaspace::is_initialized(), "metaspace should be initialized");
+  // Note: it will change flags
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  // Cannot assert, unverified entry point counts instructions (see .ad file)
+  // vtableStubs also counts instructions in pd_code_size_limit.
+  // Also do not verify_oop as this is called by verify_oop.
+  if (Universe::narrow_klass_shift() != 0) {
+    assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+    assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
+    leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
+  } else {
+    assert (Universe::narrow_klass_base() == NULL, "sanity");
+    if (dst != src) {
+      movq(dst, src);
+    }
+  }
+}
+
 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
   assert (UseCompressedOops, "should only be used for compressed headers");
   assert (Universe::heap() != NULL, "java heap should be initialized");
@@ -10293,6 +10356,22 @@ void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
   mov_narrow_oop(dst, oop_index, rspec);
 }
 
+void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int klass_index = oop_recorder()->find_index(k);
+  RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+  mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+
+void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int klass_index = oop_recorder()->find_index(k);
+  RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+  mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+
 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
   assert (UseCompressedOops, "should only be used for compressed headers");
   assert (Universe::heap() != NULL, "java heap should be initialized");
@@ -10311,9 +10390,25 @@ void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 }
 
+void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int klass_index = oop_recorder()->find_index(k);
+  RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+  Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+
+void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
+  assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+  assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+  int klass_index = oop_recorder()->find_index(k);
+  RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+  Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+
 void MacroAssembler::reinit_heapbase() {
-  if (UseCompressedOops) {
-    movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
+  if (UseCompressedOops || UseCompressedKlassPointers) {
+    movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
   }
 }
 #endif // _LP64
diff --git a/src/cpu/x86/vm/assembler_x86.hpp b/src/cpu/x86/vm/assembler_x86.hpp
index dc98fa8f6..c936e13f5 100644
--- a/src/cpu/x86/vm/assembler_x86.hpp
+++ b/src/cpu/x86/vm/assembler_x86.hpp
@@ -2083,6 +2083,15 @@ class MacroAssembler: public Assembler {
   void cmp_narrow_oop(Register dst, jobject obj);
   void cmp_narrow_oop(Address dst, jobject obj);
 
+  void encode_klass_not_null(Register r);
+  void decode_klass_not_null(Register r);
+  void encode_klass_not_null(Register dst, Register src);
+  void decode_klass_not_null(Register dst, Register src);
+  void set_narrow_klass(Register dst, Klass* k);
+  void set_narrow_klass(Address dst, Klass* k);
+  void cmp_narrow_klass(Register dst, Klass* k);
+  void cmp_narrow_klass(Address dst, Klass* k);
+
   // if heap base register is used - reinit it with the correct value
   void reinit_heapbase();
 
diff --git a/src/cpu/x86/vm/c1_FrameMap_x86.hpp b/src/cpu/x86/vm/c1_FrameMap_x86.hpp
index 75b3a0d8e..ea5d6d39d 100644
--- a/src/cpu/x86/vm/c1_FrameMap_x86.hpp
+++ b/src/cpu/x86/vm/c1_FrameMap_x86.hpp
@@ -148,7 +148,7 @@
 
   static int adjust_reg_range(int range) {
     // Reduce the number of available regs (to free r12) in case of compressed oops
-    if (UseCompressedOops) return range - 1;
+    if (UseCompressedOops || UseCompressedKlassPointers) return range - 1;
     return range;
   }
 
diff --git a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
index a0c4b6370..684ae866e 100644
--- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
+++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp
@@ -340,7 +340,7 @@ int LIR_Assembler::check_icache() {
   Register receiver = FrameMap::receiver_opr->as_register();
   Register ic_klass = IC_Klass;
   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
-  const bool do_post_padding = VerifyOops || UseCompressedOops;
+  const bool do_post_padding = VerifyOops || UseCompressedKlassPointers;
   if (!do_post_padding) {
     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
     while ((__ offset() + ic_cmp_size) % CodeEntryAlignment != 0) {
@@ -1262,7 +1262,11 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
       break;
 
     case T_ADDRESS:
-      __ movptr(dest->as_register(), from_addr);
+      if (UseCompressedKlassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
+        __ movl(dest->as_register(), from_addr);
+      } else {
+        __ movptr(dest->as_register(), from_addr);
+      }
       break;
     case T_INT:
       __ movl(dest->as_register(), from_addr);
@@ -1364,6 +1368,12 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
     }
 #endif
     __ verify_oop(dest->as_register());
+  } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
+#ifdef _LP64
+    if (UseCompressedKlassPointers) {
+      __ decode_klass_not_null(dest->as_register());
+    }
+#endif
   }
 }
 
@@ -1705,7 +1715,7 @@ void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, L
   } else if (obj == klass_RInfo) {
     klass_RInfo = dst;
   }
-  if (k->is_loaded() && !UseCompressedOops) {
+  if (k->is_loaded() && !UseCompressedKlassPointers) {
     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
   } else {
     Rtmp1 = op->tmp3()->as_register();
@@ -3446,7 +3456,7 @@ void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
     __ mov_metadata(tmp, default_type->constant_encoding());
 #ifdef _LP64
     if (UseCompressedKlassPointers) {
-      __ encode_heap_oop(tmp);
+      __ encode_klass_not_null(tmp);
     }
 #endif
 
diff --git a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
index d185613c2..5ef148619 100644
--- a/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
+++ b/src/cpu/x86/vm/c1_LIRGenerator_x86.cpp
@@ -1166,7 +1166,7 @@ void LIRGenerator::do_CheckCast(CheckCast* x) {
   }
   LIR_Opr reg = rlock_result(x);
   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
-  if (!x->klass()->is_loaded() || UseCompressedOops) {
+  if (!x->klass()->is_loaded() || UseCompressedKlassPointers) {
     tmp3 = new_register(objectType);
   }
   __ checkcast(reg, obj.result(), x->klass(),
@@ -1188,7 +1188,7 @@ void LIRGenerator::do_InstanceOf(InstanceOf* x) {
   }
   obj.load_item();
   LIR_Opr tmp3 = LIR_OprFact::illegalOpr;
-  if (!x->klass()->is_loaded() || UseCompressedOops) {
+  if (!x->klass()->is_loaded() || UseCompressedKlassPointers) {
     tmp3 = new_register(objectType);
   }
   __ instanceof(reg, obj.result(), x->klass(),
diff --git a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
index c1eb9d0f2..d9ae6ce5d 100644
--- a/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
+++ b/src/cpu/x86/vm/c1_MacroAssembler_x86.cpp
@@ -159,7 +159,7 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
 #ifdef _LP64
   if (UseCompressedKlassPointers) { // Take care not to kill klass
     movptr(t1, klass);
-    encode_heap_oop_not_null(t1);
+    encode_klass_not_null(t1);
     movl(Address(obj, oopDesc::klass_offset_in_bytes()), t1);
   } else
 #endif
diff --git a/src/cpu/x86/vm/methodHandles_x86.cpp b/src/cpu/x86/vm/methodHandles_x86.cpp
index fd01176d9..88ec6b719 100644
--- a/src/cpu/x86/vm/methodHandles_x86.cpp
+++ b/src/cpu/x86/vm/methodHandles_x86.cpp
@@ -209,8 +209,6 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
   Register rcx_mh     = rcx;   // MH receiver; dies quickly and is recycled
   Register rbx_method = rbx;   // eventual target of this invocation
 
-  address code_start = __ pc();
-
   // here's where control starts out:
   __ align(CodeEntryAlignment);
   address entry_point = __ pc();
@@ -251,23 +249,7 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
 
   // rdx_first_arg_addr is live!
 
-  if (TraceMethodHandles) {
-    const char* name = vmIntrinsics::name_at(iid);
-    if (*name == '_')  name += 1;
-    const size_t len = strlen(name) + 50;
-    char* qname = NEW_C_HEAP_ARRAY(char, len, mtInternal);
-    const char* suffix = "";
-    if (vmIntrinsics::method_for(iid) == NULL ||
-        !vmIntrinsics::method_for(iid)->access_flags().is_public()) {
-      if (is_signature_polymorphic_static(iid))
-        suffix = "/static";
-      else
-        suffix = "/private";
-    }
-    jio_snprintf(qname, len, "MethodHandle::interpreter_entry::%s%s", name, suffix);
-    // note: stub look for mh in rcx
-    trace_method_handle(_masm, qname);
-  }
+  trace_method_handle_interpreter_entry(_masm, iid);
 
   if (iid == vmIntrinsics::_invokeBasic) {
     generate_method_handle_dispatch(_masm, iid, rcx_mh, noreg, not_for_compiler_entry);
@@ -287,14 +269,6 @@ address MethodHandles::generate_method_handle_interpreter_entry(MacroAssembler*
     generate_method_handle_dispatch(_masm, iid, rcx_recv, rbx_member, not_for_compiler_entry);
   }
 
-  if (PrintMethodHandleStubs) {
-    address code_end = __ pc();
-    tty->print_cr("--------");
-    tty->print_cr("method handle interpreter entry for %s", vmIntrinsics::name_at(iid));
-    Disassembler::decode(code_start, code_end);
-    tty->cr();
-  }
-
   return entry_point;
 }
 
diff --git a/src/cpu/x86/vm/methodHandles_x86.hpp b/src/cpu/x86/vm/methodHandles_x86.hpp
index 62342eefb..bb333781a 100644
--- a/src/cpu/x86/vm/methodHandles_x86.hpp
+++ b/src/cpu/x86/vm/methodHandles_x86.hpp
@@ -55,8 +55,6 @@ public:
                                   Register temp2,
                                   bool for_compiler_entry);
 
-  static void trace_method_handle(MacroAssembler* _masm, const char* adaptername) PRODUCT_RETURN;
-
   static Register saved_last_sp_register() {
     // Should be in sharedRuntime, not here.
     return LP64_ONLY(r13) NOT_LP64(rsi);
diff --git a/src/cpu/x86/vm/vtableStubs_x86_64.cpp b/src/cpu/x86/vm/vtableStubs_x86_64.cpp
index 578023ab4..867ebfa15 100644
--- a/src/cpu/x86/vm/vtableStubs_x86_64.cpp
+++ b/src/cpu/x86/vm/vtableStubs_x86_64.cpp
@@ -212,11 +212,11 @@ int VtableStub::pd_code_size_limit(bool is_vtable_stub) {
   if (is_vtable_stub) {
     // Vtable stub size
     return (DebugVtables ? 512 : 24) + (CountCompiledCalls ? 13 : 0) +
-           (UseCompressedOops ? 16 : 0);  // 1 leaq can be 3 bytes + 1 long
+           (UseCompressedKlassPointers ? 16 : 0);  // 1 leaq can be 3 bytes + 1 long
   } else {
     // Itable stub size
     return (DebugVtables ? 512 : 74) + (CountCompiledCalls ? 13 : 0) +
-           (UseCompressedOops ? 32 : 0);  // 2 leaqs
+           (UseCompressedKlassPointers ? 32 : 0);  // 2 leaqs
   }
   // In order to tune these parameters, run the JVM with VM options
   // +PrintMiscellaneous and +WizardMode to see information about
diff --git a/src/cpu/x86/vm/x86_32.ad b/src/cpu/x86/vm/x86_32.ad
index e3afe6b3a..dafac776c 100644
--- a/src/cpu/x86/vm/x86_32.ad
+++ b/src/cpu/x86/vm/x86_32.ad
@@ -1424,6 +1424,11 @@ bool Matcher::narrow_oop_use_complex_address() {
   return true;
 }
 
+bool Matcher::narrow_klass_use_complex_address() {
+  ShouldNotCallThis();
+  return true;
+}
+
 
 // Is it better to copy float constants, or load them directly from memory?
 // Intel can load a float constant from a direct address, requiring no
@@ -1553,9 +1558,6 @@ const RegMask Matcher::method_handle_invoke_SP_save_mask() {
 // Returns true if the high 32 bits of the value is known to be zero.
 bool is_operand_hi32_zero(Node* n) {
   int opc = n->Opcode();
-  if (opc == Op_LoadUI2L) {
-    return true;
-  }
   if (opc == Op_AndL) {
     Node* o2 = n->in(2);
     if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
@@ -6147,8 +6149,8 @@ instruct loadI2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
 %}
 
 // Load Unsigned Integer into Long Register
-instruct loadUI2L(eRegL dst, memory mem, eFlagsReg cr) %{
-  match(Set dst (LoadUI2L mem));
+instruct loadUI2L(eRegL dst, memory mem, immL_32bits mask, eFlagsReg cr) %{
+  match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
   effect(KILL cr);
 
   ins_cost(250);
diff --git a/src/cpu/x86/vm/x86_64.ad b/src/cpu/x86/vm/x86_64.ad
index dc2d4d89d..5b05ec222 100644
--- a/src/cpu/x86/vm/x86_64.ad
+++ b/src/cpu/x86/vm/x86_64.ad
@@ -1409,10 +1409,10 @@ uint reloc_java_to_interp()
 #ifndef PRODUCT
 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 {
-  if (UseCompressedOops) {
+  if (UseCompressedKlassPointers) {
     st->print_cr("movl    rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
-    if (Universe::narrow_oop_shift() != 0) {
-      st->print_cr("\tdecode_heap_oop_not_null rscratch1, rscratch1");
+    if (Universe::narrow_klass_shift() != 0) {
+      st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
     }
     st->print_cr("\tcmpq    rax, rscratch1\t # Inline cache check");
   } else {
@@ -1428,7 +1428,7 @@ void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
 {
   MacroAssembler masm(&cbuf);
   uint insts_size = cbuf.insts_size();
-  if (UseCompressedOops) {
+  if (UseCompressedKlassPointers) {
     masm.load_klass(rscratch1, j_rarg0);
     masm.cmpptr(rax, rscratch1);
   } else {
@@ -1576,6 +1576,11 @@ bool Matcher::narrow_oop_use_complex_address() {
   return (LogMinObjAlignmentInBytes <= 3);
 }
 
+bool Matcher::narrow_klass_use_complex_address() {
+  assert(UseCompressedKlassPointers, "only for compressed klass code");
+  return (LogKlassAlignmentInBytes <= 3);
+}
+
 // Is it better to copy float constants, or load them directly from
 // memory?  Intel can load a float constant from a direct address,
 // requiring no extra registers.  Most RISCs will have to materialize
@@ -3139,6 +3144,14 @@ operand immN() %{
   interface(CONST_INTER);
 %}
 
+operand immNKlass() %{
+  match(ConNKlass);
+
+  op_cost(10);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // NULL Pointer Immediate
 operand immN0() %{
   predicate(n->get_narrowcon() == 0);
@@ -4038,6 +4051,145 @@ operand indPosIndexScaleOffsetNarrow(rRegN reg, immL32 off, rRegI idx, immI2 sca
   %}
 %}
 
+operand indirectNarrowKlass(rRegN reg)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(DecodeNKlass reg);
+
+  format %{ "[$reg]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index(0x4);
+    scale(0x0);
+    disp(0x0);
+  %}
+%}
+
+operand indOffset8NarrowKlass(rRegN reg, immL8 off)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (DecodeNKlass reg) off);
+
+  format %{ "[$reg + $off (8-bit)]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index(0x4);
+    scale(0x0);
+    disp($off);
+  %}
+%}
+
+operand indOffset32NarrowKlass(rRegN reg, immL32 off)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (DecodeNKlass reg) off);
+
+  format %{ "[$reg + $off (32-bit)]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index(0x4);
+    scale(0x0);
+    disp($off);
+  %}
+%}
+
+operand indIndexOffsetNarrowKlass(rRegN reg, rRegL lreg, immL32 off)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (AddP (DecodeNKlass reg) lreg) off);
+
+  op_cost(10);
+  format %{"[$reg + $off + $lreg]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index($lreg);
+    scale(0x0);
+    disp($off);
+  %}
+%}
+
+operand indIndexNarrowKlass(rRegN reg, rRegL lreg)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (DecodeNKlass reg) lreg);
+
+  op_cost(10);
+  format %{"[$reg + $lreg]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index($lreg);
+    scale(0x0);
+    disp(0x0);
+  %}
+%}
+
+operand indIndexScaleNarrowKlass(rRegN reg, rRegL lreg, immI2 scale)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (DecodeNKlass reg) (LShiftL lreg scale));
+
+  op_cost(10);
+  format %{"[$reg + $lreg << $scale]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index($lreg);
+    scale($scale);
+    disp(0x0);
+  %}
+%}
+
+operand indIndexScaleOffsetNarrowKlass(rRegN reg, immL32 off, rRegL lreg, immI2 scale)
+%{
+  predicate(Universe::narrow_klass_shift() == 0);
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (AddP (DecodeNKlass reg) (LShiftL lreg scale)) off);
+
+  op_cost(10);
+  format %{"[$reg + $off + $lreg << $scale]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index($lreg);
+    scale($scale);
+    disp($off);
+  %}
+%}
+
+operand indCompressedKlassOffset(rRegN reg, immL32 off) %{
+  predicate(UseCompressedKlassPointers && (Universe::narrow_klass_shift() == Address::times_8));
+  constraint(ALLOC_IN_RC(ptr_reg));
+  match(AddP (DecodeNKlass reg) off);
+
+  op_cost(10);
+  format %{"[R12 + $reg << 3 + $off] (compressed klass addressing)" %}
+  interface(MEMORY_INTER) %{
+    base(0xc); // R12
+    index($reg);
+    scale(0x3);
+    disp($off);
+  %}
+%}
+
+operand indPosIndexScaleOffsetNarrowKlass(rRegN reg, immL32 off, rRegI idx, immI2 scale)
+%{
+  constraint(ALLOC_IN_RC(ptr_reg));
+  predicate(Universe::narrow_klass_shift() == 0 && n->in(2)->in(3)->in(1)->as_Type()->type()->is_long()->_lo >= 0);
+  match(AddP (AddP (DecodeNKlass reg) (LShiftL (ConvI2L idx) scale)) off);
+
+  op_cost(10);
+  format %{"[$reg + $off + $idx << $scale]" %}
+  interface(MEMORY_INTER) %{
+    base($reg);
+    index($idx);
+    scale($scale);
+    disp($off);
+  %}
+%}
 
 //----------Special Memory Operands--------------------------------------------
 // Stack Slot Operand - This operand is used for loading and storing temporary
@@ -4209,7 +4361,11 @@ opclass memory(indirect, indOffset8, indOffset32, indIndexOffset, indIndex,
                indCompressedOopOffset,
                indirectNarrow, indOffset8Narrow, indOffset32Narrow,
                indIndexOffsetNarrow, indIndexNarrow, indIndexScaleNarrow,
-               indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow);
+               indIndexScaleOffsetNarrow, indPosIndexScaleOffsetNarrow,
+               indCompressedKlassOffset,
+               indirectNarrowKlass, indOffset8NarrowKlass, indOffset32NarrowKlass,
+               indIndexOffsetNarrowKlass, indIndexNarrowKlass, indIndexScaleNarrowKlass,
+               indIndexScaleOffsetNarrowKlass, indPosIndexScaleOffsetNarrowKlass);
 
 //----------PIPELINE-----------------------------------------------------------
 // Rules which define the behavior of the target architectures pipeline.
@@ -5044,9 +5200,9 @@ instruct loadI2L_immI(rRegL dst, memory mem, immI mask, rFlagsReg cr) %{
 %}
 
 // Load Unsigned Integer into Long Register
-instruct loadUI2L(rRegL dst, memory mem)
+instruct loadUI2L(rRegL dst, memory mem, immL_32bits mask) 
 %{
-  match(Set dst (LoadUI2L mem));
+  match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
 
   ins_cost(125);
   format %{ "movl    $dst, $mem\t# uint -> long" %}
@@ -5469,6 +5625,22 @@ instruct loadConN(rRegN dst, immN src) %{
   ins_pipe(ialu_reg_fat); // XXX
 %}
 
+instruct loadConNKlass(rRegN dst, immNKlass src) %{
+  match(Set dst src);
+
+  ins_cost(125);
+  format %{ "movl    $dst, $src\t# compressed klass ptr" %}
+  ins_encode %{
+    address con = (address)$src$$constant;
+    if (con == NULL) {
+      ShouldNotReachHere();
+    } else {
+      __ set_narrow_klass($dst$$Register, (Klass*)$src$$constant);
+    }
+  %}
+  ins_pipe(ialu_reg_fat); // XXX
+%}
+
 instruct loadConF0(regF dst, immF0 src)
 %{
   match(Set dst src);
@@ -5738,7 +5910,7 @@ instruct storeP(memory mem, any_RegP src)
 
 instruct storeImmP0(memory mem, immP0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreP mem zero));
 
   ins_cost(125); // XXX
@@ -5774,9 +5946,21 @@ instruct storeN(memory mem, rRegN src)
   ins_pipe(ialu_mem_reg);
 %}
 
+instruct storeNKlass(memory mem, rRegN src)
+%{
+  match(Set mem (StoreNKlass mem src));
+
+  ins_cost(125); // XXX
+  format %{ "movl    $mem, $src\t# compressed klass ptr" %}
+  ins_encode %{
+    __ movl($mem$$Address, $src$$Register);
+  %}
+  ins_pipe(ialu_mem_reg);
+%}
+
 instruct storeImmN0(memory mem, immN0 zero)
 %{
-  predicate(Universe::narrow_oop_base() == NULL);
+  predicate(Universe::narrow_oop_base() == NULL && Universe::narrow_klass_base() == NULL);
   match(Set mem (StoreN mem zero));
 
   ins_cost(125); // XXX
@@ -5804,10 +5988,22 @@ instruct storeImmN(memory mem, immN src)
   ins_pipe(ialu_mem_imm);
 %}
 
+instruct storeImmNKlass(memory mem, immNKlass src)
+%{
+  match(Set mem (StoreNKlass mem src));
+
+  ins_cost(150); // XXX
+  format %{ "movl    $mem, $src\t# compressed klass ptr" %}
+  ins_encode %{
+    __ set_narrow_klass($mem$$Address, (Klass*)$src$$constant);
+  %}
+  ins_pipe(ialu_mem_imm);
+%}
+
 // Store Integer Immediate
 instruct storeImmI0(memory mem, immI0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreI mem zero));
 
   ins_cost(125); // XXX
@@ -5832,7 +6028,7 @@ instruct storeImmI(memory mem, immI src)
 // Store Long Immediate
 instruct storeImmL0(memory mem, immL0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreL mem zero));
 
   ins_cost(125); // XXX
@@ -5857,7 +6053,7 @@ instruct storeImmL(memory mem, immL32 src)
 // Store Short/Char Immediate
 instruct storeImmC0(memory mem, immI0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreC mem zero));
 
   ins_cost(125); // XXX
@@ -5883,7 +6079,7 @@ instruct storeImmI16(memory mem, immI16 src)
 // Store Byte Immediate
 instruct storeImmB0(memory mem, immI0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreB mem zero));
 
   ins_cost(125); // XXX
@@ -5908,7 +6104,7 @@ instruct storeImmB(memory mem, immI8 src)
 // Store CMS card-mark Immediate
 instruct storeImmCM0_reg(memory mem, immI0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreCM mem zero));
 
   ins_cost(125); // XXX
@@ -5946,7 +6142,7 @@ instruct storeF(memory mem, regF src)
 // Store immediate Float value (it is faster than store from XMM register)
 instruct storeF0(memory mem, immF0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreF mem zero));
 
   ins_cost(25); // XXX
@@ -5996,7 +6192,7 @@ instruct storeD0_imm(memory mem, immD0 src)
 
 instruct storeD0(memory mem, immD0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set mem (StoreD mem zero));
 
   ins_cost(25); // XXX
@@ -6482,6 +6678,32 @@ instruct decodeHeapOop_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
   ins_pipe(ialu_reg_long);
 %}
 
+instruct encodeKlass_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
+  match(Set dst (EncodePKlass src));
+  effect(KILL cr);
+  format %{ "encode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    __ encode_klass_not_null($dst$$Register, $src$$Register);
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
+instruct decodeKlass_not_null(rRegP dst, rRegN src, rFlagsReg cr) %{
+  match(Set dst (DecodeNKlass src));
+  effect(KILL cr);
+  format %{ "decode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    if (s != d) {
+      __ decode_klass_not_null(d, s);
+    } else {
+      __ decode_klass_not_null(d);
+    }
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
 
 //----------Conditional Move---------------------------------------------------
 // Jump
@@ -10452,7 +10674,7 @@ instruct testP_mem(rFlagsReg cr, memory op, immP0 zero)
 
 instruct testP_mem_reg0(rFlagsReg cr, memory mem, immP0 zero)
 %{
-  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL));
+  predicate(UseCompressedOops && (Universe::narrow_oop_base() == NULL) && (Universe::narrow_klass_base() == NULL));
   match(Set cr (CmpP (LoadP mem) zero));
 
   format %{ "cmpq    R12, $mem\t# ptr (R12_heapbase==0)" %}
@@ -10503,6 +10725,27 @@ instruct compN_mem_imm(rFlagsRegU cr, memory mem, immN src)
   ins_pipe(ialu_cr_reg_mem);
 %}
 
+instruct compN_rReg_imm_klass(rFlagsRegU cr, rRegN op1, immNKlass op2) %{
+  match(Set cr (CmpN op1 op2));
+
+  format %{ "cmpl    $op1, $op2\t# compressed klass ptr" %}
+  ins_encode %{
+    __ cmp_narrow_klass($op1$$Register, (Klass*)$op2$$constant);
+  %}
+  ins_pipe(ialu_cr_reg_imm);
+%}
+
+instruct compN_mem_imm_klass(rFlagsRegU cr, memory mem, immNKlass src)
+%{
+  match(Set cr (CmpN src (LoadNKlass mem)));
+
+  format %{ "cmpl    $mem, $src\t# compressed klass ptr" %}
+  ins_encode %{
+    __ cmp_narrow_klass($mem$$Address, (Klass*)$src$$constant);
+  %}
+  ins_pipe(ialu_cr_reg_mem);
+%}
+
 instruct testN_reg(rFlagsReg cr, rRegN src, immN0 zero) %{
   match(Set cr (CmpN src zero));
 
@@ -10526,7 +10769,7 @@ instruct testN_mem(rFlagsReg cr, memory mem, immN0 zero)
 
 instruct testN_mem_reg0(rFlagsReg cr, memory mem, immN0 zero)
 %{
-  predicate(Universe::narrow_oop_base() == NULL);
+  predicate(Universe::narrow_oop_base() == NULL && (Universe::narrow_klass_base() == NULL));
   match(Set cr (CmpN (LoadN mem) zero));
 
   format %{ "cmpl    R12, $mem\t# compressed ptr (R12_heapbase==0)" %}