7107135: Stack guard pages are no more protected after loading a shared library with executable stack

Summary: Detect the execstack attribute of the loaded library and attempt to fix the stack guard using Safepoint op.
Reviewed-by: dholmes, zgu
Contributed-by: ioi.lam@oracle.com

2 files changed, 34 insertions, 0 deletions

diff --git a/src/share/vm/utilities/elfFile.cpp b/src/share/vm/utilities/elfFile.cpp
index 155a78f5d..9fba1c39c 100644
--- a/src/share/vm/utilities/elfFile.cpp
+++ b/src/share/vm/utilities/elfFile.cpp
@@ -197,4 +197,28 @@ ElfStringTable* ElfFile::get_string_table(int index) {
   return NULL;
 }
 
+#ifdef LINUX
+bool ElfFile::specifies_noexecstack() {
+  Elf_Phdr phdr;
+  if (!m_file)  return true;
+
+  if (!fseek(m_file, m_elfHdr.e_phoff, SEEK_SET)) {
+    for (int index = 0; index < m_elfHdr.e_phnum; index ++) {
+      if (fread((void*)&phdr, sizeof(Elf_Phdr), 1, m_file) != 1) {
+        m_status = NullDecoder::file_invalid;
+        return false;
+      }
+      if (phdr.p_type == PT_GNU_STACK) {
+        if (phdr.p_flags == (PF_R | PF_W))  {
+          return true;
+        } else {
+          return false;
+        }
+      }
+    }
+  }
+  return false;
+}
+#endif
+
 #endif // _WINDOWS
diff --git a/src/share/vm/utilities/elfFile.hpp b/src/share/vm/utilities/elfFile.hpp
index 60f2a8804..d6aa886bb 100644
--- a/src/share/vm/utilities/elfFile.hpp
+++ b/src/share/vm/utilities/elfFile.hpp
@@ -43,6 +43,7 @@ typedef Elf64_Addr      Elf_Addr;
 
 typedef Elf64_Ehdr      Elf_Ehdr;
 typedef Elf64_Shdr      Elf_Shdr;
+typedef Elf64_Phdr      Elf_Phdr;
 typedef Elf64_Sym       Elf_Sym;
 
 #if !defined(_ALLBSD_SOURCE) || defined(__APPLE__)
@@ -59,6 +60,7 @@ typedef Elf32_Addr      Elf_Addr;
 
 typedef Elf32_Ehdr      Elf_Ehdr;
 typedef Elf32_Shdr      Elf_Shdr;
+typedef Elf32_Phdr      Elf_Phdr;
 typedef Elf32_Sym       Elf_Sym;
 
 #if !defined(_ALLBSD_SOURCE) || defined(__APPLE__)
@@ -123,6 +125,14 @@ protected:
    ElfFile*  next() const { return m_next; }
    void set_next(ElfFile* file) { m_next = file; }
 
+ public:
+  // Returns true if the elf file is marked NOT to require an executable stack,
+  // or if the file could not be opened.
+  // Returns false if the elf file requires an executable stack, the stack flag
+  // is not set at all, or if the file can not be read.
+  // On systems other than linux it always returns false.
+  bool specifies_noexecstack() NOT_LINUX({ return false; });
+
  protected:
     ElfFile*         m_next;