8003420: NPG: make new GC root for pd_set

Summary: Move protection domain oops from system dictionary entries into a seperate set; the system dictionary references entries in that set now. This allows fast iteration during non-classunloading garbage collection. Implementation based on initial prototype from Ioi Lam (iklam).
Reviewed-by: coleenp, iklam

7 files changed, 403 insertions, 40 deletions

diff --git a/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainCacheEntry.java b/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainCacheEntry.java
new file mode 100644
index 000000000..2c9e736ec
--- /dev/null
+++ b/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainCacheEntry.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.memory;
+
+import java.util.*;
+import sun.jvm.hotspot.debugger.*;
+import sun.jvm.hotspot.oops.*;
+import sun.jvm.hotspot.runtime.*;
+import sun.jvm.hotspot.types.*;
+
+public class ProtectionDomainCacheEntry extends VMObject {
+  private static sun.jvm.hotspot.types.OopField protectionDomainField;
+
+  static {
+    VM.registerVMInitializedObserver(new Observer() {
+        public void update(Observable o, Object data) {
+          initialize(VM.getVM().getTypeDataBase());
+        }
+      });
+  }
+
+  private static synchronized void initialize(TypeDataBase db) {
+    Type type = db.lookupType("ProtectionDomainCacheEntry");
+    protectionDomainField = type.getOopField("_literal");
+  }
+
+  public ProtectionDomainCacheEntry(Address addr) {
+    super(addr);
+  }
+
+  public Oop protectionDomain() {
+    return VM.getVM().getObjectHeap().newOop(protectionDomainField.getValue(addr));
+  }
+}
diff --git a/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainEntry.java b/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainEntry.java
index de2da04f9..27aa4e9f4 100644
--- a/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainEntry.java
+++ b/agent/src/share/classes/sun/jvm/hotspot/memory/ProtectionDomainEntry.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,7 @@ import sun.jvm.hotspot.types.*;
 
 public class ProtectionDomainEntry extends VMObject {
   private static AddressField nextField;
-  private static sun.jvm.hotspot.types.OopField protectionDomainField;
+  private static AddressField pdCacheField;
 
   static {
     VM.registerVMInitializedObserver(new Observer() {
@@ -46,7 +46,7 @@ public class ProtectionDomainEntry extends VMObject {
     Type type = db.lookupType("ProtectionDomainEntry");
 
     nextField = type.getAddressField("_next");
-    protectionDomainField = type.getOopField("_protection_domain");
+    pdCacheField = type.getAddressField("_pd_cache");
   }
 
   public ProtectionDomainEntry(Address addr) {
@@ -54,10 +54,12 @@ public class ProtectionDomainEntry extends VMObject {
   }
 
   public ProtectionDomainEntry next() {
-    return (ProtectionDomainEntry) VMObjectFactory.newObject(ProtectionDomainEntry.class, addr);
+    return (ProtectionDomainEntry) VMObjectFactory.newObject(ProtectionDomainEntry.class, nextField.getValue(addr));
   }
 
   public Oop protectionDomain() {
-    return VM.getVM().getObjectHeap().newOop(protectionDomainField.getValue(addr));
+    ProtectionDomainCacheEntry pd_cache = (ProtectionDomainCacheEntry)
+      VMObjectFactory.newObject(ProtectionDomainCacheEntry.class, pdCacheField.getValue(addr));
+    return pd_cache.protectionDomain();
   }
 }
diff --git a/src/share/vm/classfile/dictionary.cpp b/src/share/vm/classfile/dictionary.cpp
index 26e06c8a0..dbb85e539 100644
--- a/src/share/vm/classfile/dictionary.cpp
+++ b/src/share/vm/classfile/dictionary.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "classfile/dictionary.hpp"
 #include "classfile/systemDictionary.hpp"
+#include "memory/iterator.hpp"
 #include "oops/oop.inline.hpp"
 #include "prims/jvmtiRedefineClassesTrace.hpp"
 #include "utilities/hashtable.inline.hpp"
@@ -38,17 +39,21 @@ Dictionary::Dictionary(int table_size)
   : TwoOopHashtable<Klass*, mtClass>(table_size, sizeof(DictionaryEntry)) {
   _current_class_index = 0;
   _current_class_entry = NULL;
+  _pd_cache_table = new ProtectionDomainCacheTable(defaultProtectionDomainCacheSize);
 };
 
 
-
 Dictionary::Dictionary(int table_size, HashtableBucket<mtClass>* t,
                        int number_of_entries)
   : TwoOopHashtable<Klass*, mtClass>(table_size, sizeof(DictionaryEntry), t, number_of_entries) {
   _current_class_index = 0;
   _current_class_entry = NULL;
+  _pd_cache_table = new ProtectionDomainCacheTable(defaultProtectionDomainCacheSize);
 };
 
+ProtectionDomainCacheEntry* Dictionary::cache_get(oop protection_domain) {
+  return _pd_cache_table->get(protection_domain);
+}
 
 DictionaryEntry* Dictionary::new_entry(unsigned int hash, Klass* klass,
                                        ClassLoaderData* loader_data) {
@@ -105,11 +110,12 @@ bool DictionaryEntry::contains_protection_domain(oop protection_domain) const {
 }
 
 
-void DictionaryEntry::add_protection_domain(oop protection_domain) {
+void DictionaryEntry::add_protection_domain(Dictionary* dict, oop protection_domain) {
   assert_locked_or_safepoint(SystemDictionary_lock);
   if (!contains_protection_domain(protection_domain)) {
+    ProtectionDomainCacheEntry* entry = dict->cache_get(protection_domain);
     ProtectionDomainEntry* new_head =
-                new ProtectionDomainEntry(protection_domain, _pd_set);
+                new ProtectionDomainEntry(entry, _pd_set);
     // Warning: Preserve store ordering.  The SystemDictionary is read
     //          without locks.  The new ProtectionDomainEntry must be
     //          complete before other threads can be allowed to see it
@@ -193,7 +199,10 @@ bool Dictionary::do_unloading() {
 
 
 void Dictionary::always_strong_oops_do(OopClosure* blk) {
-  // Follow all system classes and temporary placeholders in dictionary
+  // Follow all system classes and temporary placeholders in dictionary; only
+  // protection domain oops contain references into the heap. In a first
+  // pass over the system dictionary determine which need to be treated as
+  // strongly reachable and mark them as such.
   for (int index = 0; index < table_size(); index++) {
     for (DictionaryEntry *probe = bucket(index);
                           probe != NULL;
@@ -201,10 +210,13 @@ void Dictionary::always_strong_oops_do(OopClosure* blk) {
       Klass* e = probe->klass();
       ClassLoaderData* loader_data = probe->loader_data();
       if (is_strongly_reachable(loader_data, e)) {
-        probe->protection_domain_set_oops_do(blk);
+        probe->set_strongly_reachable();
       }
     }
   }
+  // Then iterate over the protection domain cache to apply the closure on the
+  // previously marked ones.
+  _pd_cache_table->always_strong_oops_do(blk);
 }
 
 
@@ -266,18 +278,12 @@ void Dictionary::classes_do(void f(Klass*, ClassLoaderData*)) {
   }
 }
 
-
 void Dictionary::oops_do(OopClosure* f) {
-  for (int index = 0; index < table_size(); index++) {
-    for (DictionaryEntry* probe = bucket(index);
-                          probe != NULL;
-                          probe = probe->next()) {
-      probe->protection_domain_set_oops_do(f);
-    }
-  }
+  // Only the protection domain oops contain references into the heap. Iterate
+  // over all of them.
+  _pd_cache_table->oops_do(f);
 }
 
-
 void Dictionary::methods_do(void f(Method*)) {
   for (int index = 0; index < table_size(); index++) {
     for (DictionaryEntry* probe = bucket(index);
@@ -292,6 +298,11 @@ void Dictionary::methods_do(void f(Method*)) {
   }
 }
 
+void Dictionary::unlink(BoolObjectClosure* is_alive) {
+  // Only the protection domain cache table may contain references to the heap
+  // that need to be unlinked.
+  _pd_cache_table->unlink(is_alive);
+}
 
 Klass* Dictionary::try_get_next_class() {
   while (true) {
@@ -306,7 +317,6 @@ Klass* Dictionary::try_get_next_class() {
   // never reached
 }
 
-
 // Add a loaded class to the system dictionary.
 // Readers of the SystemDictionary aren't always locked, so _buckets
 // is volatile. The store of the next field in the constructor is
@@ -396,7 +406,7 @@ void Dictionary::add_protection_domain(int index, unsigned int hash,
   assert(protection_domain() != NULL,
          "real protection domain should be present");
 
-  entry->add_protection_domain(protection_domain());
+  entry->add_protection_domain(this, protection_domain());
 
   assert(entry->contains_protection_domain(protection_domain()),
          "now protection domain should be present");
@@ -446,6 +456,146 @@ void Dictionary::reorder_dictionary() {
   }
 }
 
+ProtectionDomainCacheTable::ProtectionDomainCacheTable(int table_size)
+  : Hashtable<oop, mtClass>(table_size, sizeof(ProtectionDomainCacheEntry))
+{
+}
+
+void ProtectionDomainCacheTable::unlink(BoolObjectClosure* is_alive) {
+  assert(SafepointSynchronize::is_at_safepoint(), "must be");
+  for (int i = 0; i < table_size(); ++i) {
+    ProtectionDomainCacheEntry** p = bucket_addr(i);
+    ProtectionDomainCacheEntry* entry = bucket(i);
+    while (entry != NULL) {
+      if (is_alive->do_object_b(entry->literal())) {
+        p = entry->next_addr();
+      } else {
+        *p = entry->next();
+        free_entry(entry);
+      }
+      entry = *p;
+    }
+  }
+}
+
+void ProtectionDomainCacheTable::oops_do(OopClosure* f) {
+  for (int index = 0; index < table_size(); index++) {
+    for (ProtectionDomainCacheEntry* probe = bucket(index);
+                                     probe != NULL;
+                                     probe = probe->next()) {
+      probe->oops_do(f);
+    }
+  }
+}
+
+uint ProtectionDomainCacheTable::bucket_size() {
+  return sizeof(ProtectionDomainCacheEntry);
+}
+
+#ifndef PRODUCT
+void ProtectionDomainCacheTable::print() {
+  tty->print_cr("Protection domain cache table (table_size=%d, classes=%d)",
+                table_size(), number_of_entries());
+  for (int index = 0; index < table_size(); index++) {
+    for (ProtectionDomainCacheEntry* probe = bucket(index);
+                                     probe != NULL;
+                                     probe = probe->next()) {
+      probe->print();
+    }
+  }
+}
+
+void ProtectionDomainCacheEntry::print() {
+  tty->print_cr("entry "PTR_FORMAT" value "PTR_FORMAT" strongly_reachable %d next "PTR_FORMAT,
+                this, literal(), _strongly_reachable, next());
+}
+#endif
+
+void ProtectionDomainCacheTable::verify() {
+  int element_count = 0;
+  for (int index = 0; index < table_size(); index++) {
+    for (ProtectionDomainCacheEntry* probe = bucket(index);
+                                     probe != NULL;
+                                     probe = probe->next()) {
+      probe->verify();
+      element_count++;
+    }
+  }
+  guarantee(number_of_entries() == element_count,
+            "Verify of protection domain cache table failed");
+  debug_only(verify_lookup_length((double)number_of_entries() / table_size()));
+}
+
+void ProtectionDomainCacheEntry::verify() {
+  guarantee(literal()->is_oop(), "must be an oop");
+}
+
+void ProtectionDomainCacheTable::always_strong_oops_do(OopClosure* f) {
+  // the caller marked the protection domain cache entries that we need to apply
+  // the closure on. Only process them.
+  for (int index = 0; index < table_size(); index++) {
+    for (ProtectionDomainCacheEntry* probe = bucket(index);
+                                     probe != NULL;
+                                     probe = probe->next()) {
+      if (probe->is_strongly_reachable()) {
+        probe->reset_strongly_reachable();
+        probe->oops_do(f);
+      }
+    }
+  }
+}
+
+ProtectionDomainCacheEntry* ProtectionDomainCacheTable::get(oop protection_domain) {
+  unsigned int hash = compute_hash(protection_domain);
+  int index = hash_to_index(hash);
+
+  ProtectionDomainCacheEntry* entry = find_entry(index, protection_domain);
+  if (entry == NULL) {
+    entry = add_entry(index, hash, protection_domain);
+  }
+  return entry;
+}
+
+ProtectionDomainCacheEntry* ProtectionDomainCacheTable::find_entry(int index, oop protection_domain) {
+  for (ProtectionDomainCacheEntry* e = bucket(index); e != NULL; e = e->next()) {
+    if (e->protection_domain() == protection_domain) {
+      return e;
+    }
+  }
+
+  return NULL;
+}
+
+ProtectionDomainCacheEntry* ProtectionDomainCacheTable::add_entry(int index, unsigned int hash, oop protection_domain) {
+  assert_locked_or_safepoint(SystemDictionary_lock);
+  assert(index == index_for(protection_domain), "incorrect index?");
+  assert(find_entry(index, protection_domain) == NULL, "no double entry");
+
+  ProtectionDomainCacheEntry* p = new_entry(hash, protection_domain);
+  Hashtable<oop, mtClass>::add_entry(index, p);
+  return p;
+}
+
+void ProtectionDomainCacheTable::free(ProtectionDomainCacheEntry* to_delete) {
+  unsigned int hash = compute_hash(to_delete->protection_domain());
+  int index = hash_to_index(hash);
+
+  ProtectionDomainCacheEntry** p = bucket_addr(index);
+  ProtectionDomainCacheEntry* entry = bucket(index);
+  while (true) {
+    assert(entry != NULL, "sanity");
+
+    if (entry == to_delete) {
+      *p = entry->next();
+      Hashtable<oop, mtClass>::free_entry(entry);
+      break;
+    } else {
+      p = entry->next_addr();
+      entry = *p;
+    }
+  }
+}
+
 SymbolPropertyTable::SymbolPropertyTable(int table_size)
   : Hashtable<Symbol*, mtSymbol>(table_size, sizeof(SymbolPropertyEntry))
 {
@@ -532,11 +682,13 @@ void Dictionary::print() {
       tty->cr();
     }
   }
+  tty->cr();
+  _pd_cache_table->print();
+  tty->cr();
 }
 
 #endif
 
-
 void Dictionary::verify() {
   guarantee(number_of_entries() >= 0, "Verify of system dictionary failed");
 
@@ -563,5 +715,7 @@ void Dictionary::verify() {
   guarantee(number_of_entries() == element_count,
             "Verify of system dictionary failed");
   debug_only(verify_lookup_length((double)number_of_entries() / table_size()));
+
+  _pd_cache_table->verify();
 }
 
diff --git a/src/share/vm/classfile/dictionary.hpp b/src/share/vm/classfile/dictionary.hpp
index 53629a015..220c1d764 100644
--- a/src/share/vm/classfile/dictionary.hpp
+++ b/src/share/vm/classfile/dictionary.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,11 +27,14 @@
 
 #include "classfile/systemDictionary.hpp"
 #include "oops/instanceKlass.hpp"
-#include "oops/oop.hpp"
+#include "oops/oop.inline.hpp"
 #include "utilities/hashtable.hpp"
 
 class DictionaryEntry;
 class PSPromotionManager;
+class ProtectionDomainCacheTable;
+class ProtectionDomainCacheEntry;
+class BoolObjectClosure;
 
 //~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 // The data structure for the system dictionary (and the shared system
@@ -45,6 +48,8 @@ private:
   // pointer to the current hash table entry.
   static DictionaryEntry*       _current_class_entry;
 
+  ProtectionDomainCacheTable*   _pd_cache_table;
+
   DictionaryEntry* get_entry(int index, unsigned int hash,
                              Symbol* name, ClassLoaderData* loader_data);
 
@@ -93,6 +98,7 @@ public:
 
   void methods_do(void f(Method*));
 
+  void unlink(BoolObjectClosure* is_alive);
 
   // Classes loaded by the bootstrap loader are always strongly reachable.
   // If we're not doing class unloading, all classes are strongly reachable.
@@ -118,6 +124,7 @@ public:
   // Sharing support
   void reorder_dictionary();
 
+  ProtectionDomainCacheEntry* cache_get(oop protection_domain);
 
 #ifndef PRODUCT
   void print();
@@ -126,21 +133,112 @@ public:
 };
 
 // The following classes can be in dictionary.cpp, but we need these
-// to be in header file so that SA's vmStructs can access.
+// to be in header file so that SA's vmStructs can access them.
+class ProtectionDomainCacheEntry : public HashtableEntry<oop, mtClass> {
+  friend class VMStructs;
+ private:
+  // Flag indicating whether this protection domain entry is strongly reachable.
+  // Used during iterating over the system dictionary to remember oops that need
+  // to be updated.
+  bool _strongly_reachable;
+ public:
+  oop protection_domain() { return literal(); }
+
+  void init() {
+    _strongly_reachable = false;
+  }
+
+  ProtectionDomainCacheEntry* next() {
+    return (ProtectionDomainCacheEntry*)HashtableEntry<oop, mtClass>::next();
+  }
+
+  ProtectionDomainCacheEntry** next_addr() {
+    return (ProtectionDomainCacheEntry**)HashtableEntry<oop, mtClass>::next_addr();
+  }
+
+  void oops_do(OopClosure* f) {
+    f->do_oop(literal_addr());
+  }
+
+  void set_strongly_reachable()   { _strongly_reachable = true; }
+  bool is_strongly_reachable()    { return _strongly_reachable; }
+  void reset_strongly_reachable() { _strongly_reachable = false; }
+
+  void print() PRODUCT_RETURN;
+  void verify();
+};
+
+// The ProtectionDomainCacheTable contains all protection domain oops. The system
+// dictionary entries reference its entries instead of having references to oops
+// directly.
+// This is used to speed up system dictionary iteration: the oops in the
+// protection domain are the only ones referring the Java heap. So when there is
+// need to update these, instead of going over every entry of the system dictionary,
+// we only need to iterate over this set.
+// The amount of different protection domains used is typically magnitudes smaller
+// than the number of system dictionary entries (loaded classes).
+class ProtectionDomainCacheTable : public Hashtable<oop, mtClass> {
+  friend class VMStructs;
+private:
+  ProtectionDomainCacheEntry* bucket(int i) {
+    return (ProtectionDomainCacheEntry*) Hashtable<oop, mtClass>::bucket(i);
+  }
+
+  // The following method is not MT-safe and must be done under lock.
+  ProtectionDomainCacheEntry** bucket_addr(int i) {
+    return (ProtectionDomainCacheEntry**) Hashtable<oop, mtClass>::bucket_addr(i);
+  }
+
+  ProtectionDomainCacheEntry* new_entry(unsigned int hash, oop protection_domain) {
+    ProtectionDomainCacheEntry* entry = (ProtectionDomainCacheEntry*) Hashtable<oop, mtClass>::new_entry(hash, protection_domain);
+    entry->init();
+    return entry;
+  }
+
+  static unsigned int compute_hash(oop protection_domain) {
+    return (unsigned int)(protection_domain->identity_hash());
+  }
+
+  int index_for(oop protection_domain) {
+    return hash_to_index(compute_hash(protection_domain));
+  }
+
+  ProtectionDomainCacheEntry* add_entry(int index, unsigned int hash, oop protection_domain);
+  ProtectionDomainCacheEntry* find_entry(int index, oop protection_domain);
+
+public:
+
+  ProtectionDomainCacheTable(int table_size);
+
+  ProtectionDomainCacheEntry* get(oop protection_domain);
+  void free(ProtectionDomainCacheEntry* entry);
+
+  void unlink(BoolObjectClosure* cl);
+
+  // GC support
+  void oops_do(OopClosure* f);
+  void always_strong_oops_do(OopClosure* f);
+
+  static uint bucket_size();
+
+  void print() PRODUCT_RETURN;
+  void verify();
+};
+
 
 class ProtectionDomainEntry :public CHeapObj<mtClass> {
   friend class VMStructs;
  public:
   ProtectionDomainEntry* _next;
-  oop                    _protection_domain;
+  ProtectionDomainCacheEntry* _pd_cache;
 
-  ProtectionDomainEntry(oop protection_domain, ProtectionDomainEntry* next) {
-    _protection_domain = protection_domain;
-    _next              = next;
+  ProtectionDomainEntry(ProtectionDomainCacheEntry* pd_cache, ProtectionDomainEntry* next) {
+    _pd_cache = pd_cache;
+    _next     = next;
   }
 
   ProtectionDomainEntry* next() { return _next; }
-  oop protection_domain() { return _protection_domain; }
+  oop protection_domain() { return _pd_cache->protection_domain(); }
 };
 
 // An entry in the system dictionary, this describes a class as
@@ -151,6 +249,24 @@ class DictionaryEntry : public HashtableEntry<Klass*, mtClass> {
  private:
   // Contains the set of approved protection domains that can access
   // this system dictionary entry.
+  //
+  // This protection domain set is a set of tuples:
+  //
+  // (InstanceKlass C, initiating class loader ICL, Protection Domain PD)
+  //
+  // [Note that C.protection_domain(), which is stored in the java.lang.Class
+  // mirror of C, is NOT the same as PD]
+  //
+  // If such an entry (C, ICL, PD) exists in the table, it means that
+  // it is okay for a class Foo to reference C, where
+  //
+  //    Foo.protection_domain() == PD, and
+  //    Foo's defining class loader == ICL
+  //
+  // The usage of the PD set can be seen in SystemDictionary::validate_protection_domain()
+  // It is essentially a cache to avoid repeated Java up-calls to
+  // ClassLoader.checkPackageAccess().
+  //
   ProtectionDomainEntry* _pd_set;
   ClassLoaderData*       _loader_data;
 
@@ -158,7 +274,7 @@ class DictionaryEntry : public HashtableEntry<Klass*, mtClass> {
   // Tells whether a protection is in the approved set.
   bool contains_protection_domain(oop protection_domain) const;
   // Adds a protection domain to the approved set.
-  void add_protection_domain(oop protection_domain);
+  void add_protection_domain(Dictionary* dict, oop protection_domain);
 
   Klass* klass() const { return (Klass*)literal(); }
   Klass** klass_addr() { return (Klass**)literal_addr(); }
@@ -189,12 +305,11 @@ class DictionaryEntry : public HashtableEntry<Klass*, mtClass> {
          : contains_protection_domain(protection_domain());
   }
 
-
-  void protection_domain_set_oops_do(OopClosure* f) {
+  void set_strongly_reachable() {
     for (ProtectionDomainEntry* current = _pd_set;
                                 current != NULL;
                                 current = current->_next) {
-      f->do_oop(&(current->_protection_domain));
+      current->_pd_cache->set_strongly_reachable();
     }
   }
 
@@ -202,7 +317,7 @@ class DictionaryEntry : public HashtableEntry<Klass*, mtClass> {
     for (ProtectionDomainEntry* current = _pd_set;
                                 current != NULL;
                                 current = current->_next) {
-      current->_protection_domain->verify();
+      current->_pd_cache->protection_domain()->verify();
     }
   }
 
diff --git a/src/share/vm/classfile/systemDictionary.cpp b/src/share/vm/classfile/systemDictionary.cpp
index c0d50ca9a..b6731c2e8 100644
--- a/src/share/vm/classfile/systemDictionary.cpp
+++ b/src/share/vm/classfile/systemDictionary.cpp
@@ -1697,6 +1697,24 @@ int SystemDictionary::calculate_systemdictionary_size(int classcount) {
   return newsize;
 }
 
+#ifdef ASSERT
+class VerifySDReachableAndLiveClosure : public OopClosure {
+private:
+  BoolObjectClosure* _is_alive;
+
+  template <class T> void do_oop_work(T* p) {
+    oop obj = oopDesc::load_decode_heap_oop(p);
+    guarantee(_is_alive->do_object_b(obj), "Oop in system dictionary must be live");
+  }
+
+public:
+  VerifySDReachableAndLiveClosure(BoolObjectClosure* is_alive) : OopClosure(), _is_alive(is_alive) { }
+
+  virtual void do_oop(oop* p)       { do_oop_work(p); }
+  virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+};
+#endif
+
 // Assumes classes in the SystemDictionary are only unloaded at a safepoint
 // Note: anonymous classes are not in the SD.
 bool SystemDictionary::do_unloading(BoolObjectClosure* is_alive) {
@@ -1707,7 +1725,15 @@ bool SystemDictionary::do_unloading(BoolObjectClosure* is_alive) {
     unloading_occurred = dictionary()->do_unloading();
     constraints()->purge_loader_constraints();
     resolution_errors()->purge_resolution_errors();
-}
+  }
+  // Oops referenced by the system dictionary may get unreachable independently
+  // of the class loader (eg. cached protection domain oops). So we need to
+  // explicitly unlink them here instead of in Dictionary::do_unloading.
+  dictionary()->unlink(is_alive);
+#ifdef ASSERT
+  VerifySDReachableAndLiveClosure cl(is_alive);
+  dictionary()->oops_do(&cl);
+#endif
   return unloading_occurred;
 }
 
diff --git a/src/share/vm/runtime/vmStructs.cpp b/src/share/vm/runtime/vmStructs.cpp
index 55405b27f..1bc9e7fa1 100644
--- a/src/share/vm/runtime/vmStructs.cpp
+++ b/src/share/vm/runtime/vmStructs.cpp
@@ -714,11 +714,17 @@ typedef BinaryTreeDictionary<Metablock, FreeList> MetablockTreeDictionary;
   nonstatic_field(PlaceholderEntry,            _loader_data,                                  ClassLoaderData*)                      \
                                                                                                                                      \
   /**************************/                                                                                                       \
-  /* ProctectionDomainEntry */                                                                                                       \
+  /* ProtectionDomainEntry  */                                                                                                       \
   /**************************/                                                                                                       \
                                                                                                                                      \
   nonstatic_field(ProtectionDomainEntry,       _next,                                         ProtectionDomainEntry*)                \
-  nonstatic_field(ProtectionDomainEntry,       _protection_domain,                            oop)                                   \
+  nonstatic_field(ProtectionDomainEntry,       _pd_cache,                                     ProtectionDomainCacheEntry*)           \
+                                                                                                                                     \
+  /*******************************/                                                                                                  \
+  /* ProtectionDomainCacheEntry  */                                                                                                  \
+  /*******************************/                                                                                                  \
+                                                                                                                                     \
+  nonstatic_field(ProtectionDomainCacheEntry,  _literal,                                      oop)                                   \
                                                                                                                                      \
   /*************************/                                                                                                        \
   /* LoaderConstraintEntry */                                                                                                        \
@@ -1560,6 +1566,7 @@ typedef BinaryTreeDictionary<Metablock, FreeList> MetablockTreeDictionary;
   declare_toplevel_type(SystemDictionary)                                 \
   declare_toplevel_type(vmSymbols)                                        \
   declare_toplevel_type(ProtectionDomainEntry)                            \
+  declare_toplevel_type(ProtectionDomainCacheEntry)                       \
                                                                           \
   declare_toplevel_type(GenericGrowableArray)                             \
   declare_toplevel_type(GrowableArray<int>)                               \
diff --git a/src/share/vm/utilities/globalDefinitions.hpp b/src/share/vm/utilities/globalDefinitions.hpp
index bbee85c8b..fb1646917 100644
--- a/src/share/vm/utilities/globalDefinitions.hpp
+++ b/src/share/vm/utilities/globalDefinitions.hpp
@@ -326,12 +326,15 @@ typedef jlong  s8;
 
 const int max_method_code_size = 64*K - 1;  // JVM spec, 2nd ed. section 4.8.1 (p.134)
 
+// Default ProtectionDomainCacheSize values
+
+const int defaultProtectionDomainCacheSize = NOT_LP64(137) LP64_ONLY(2017);
 
 //----------------------------------------------------------------------------------------------------
 // Default and minimum StringTableSize values
 
 const int defaultStringTableSize = NOT_LP64(1009) LP64_ONLY(60013);
-const int minimumStringTableSize=1009;
+const int minimumStringTableSize = 1009;
 
 
 //----------------------------------------------------------------------------------------------------