8048549: [macosx] Disable usage of system menu bar if AWT is embedded in FX

Reviewed-by: serb, anthony

3 files changed, 35 insertions, 22 deletions

diff --git a/src/macosx/classes/com/apple/laf/AquaMenuBarUI.java b/src/macosx/classes/com/apple/laf/AquaMenuBarUI.java
index c884e6d19..36e703e9f 100644
--- a/src/macosx/classes/com/apple/laf/AquaMenuBarUI.java
+++ b/src/macosx/classes/com/apple/laf/AquaMenuBarUI.java
@@ -26,11 +26,14 @@
 package com.apple.laf;
 
 import java.awt.*;
+import java.security.AccessController;
 
 import javax.swing.*;
 import javax.swing.plaf.ComponentUI;
 import javax.swing.plaf.basic.BasicMenuBarUI;
 
+import sun.lwawt.macosx.LWCToolkit;
+import sun.security.action.GetBooleanAction;
 import sun.security.action.GetPropertyAction;
 
 // MenuBar implementation for Mac L&F
@@ -131,28 +134,20 @@ public class AquaMenuBarUI extends BasicMenuBarUI implements ScreenMenuBarProvid
     ScreenMenuBar fScreenMenuBar;
     boolean useScreenMenuBar = getScreenMenuBarProperty();
 
-    private static String getPrivSysProp(final String propName) {
-        return java.security.AccessController.doPrivileged(new GetPropertyAction(propName));
-    }
-
     static boolean getScreenMenuBarProperty() {
-        final String props[] = new String[]{""};
-
-        boolean useScreenMenuBar = false;
-        try {
-            props[0] = getPrivSysProp(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-
-            if (props[0] != null && props[0].equals("true")) useScreenMenuBar = true;
-            else {
-                props[0] = getPrivSysProp(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar");
-
-                if (props[0] != null && props[0].equals("true")) {
-                    System.err.println(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar has been deprecated. Please switch to " + AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
-                    useScreenMenuBar = true;
-                }
-            }
-        } catch(final Throwable t) { };
-
-        return useScreenMenuBar;
+        // Do not allow AWT to set the screen menu bar if it's embedded in another UI toolkit
+        if (LWCToolkit.isEmbedded()) return false;
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar"))) {
+            return true;
+        }
+        if (AccessController.doPrivileged(
+                new GetBooleanAction(AquaLookAndFeel.sOldPropertyPrefix + "useScreenMenuBar"))) {
+                System.err.println(AquaLookAndFeel.sOldPropertyPrefix +
+                        "useScreenMenuBar has been deprecated. Please switch to " +
+                        AquaLookAndFeel.sPropertyPrefix + "useScreenMenuBar");
+                return true;
+        }
+        return false;
     }
 }
diff --git a/src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java b/src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java
index 86b65a00f..eb9894810 100644
--- a/src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java
+++ b/src/macosx/classes/sun/lwawt/macosx/LWCToolkit.java
@@ -784,6 +784,13 @@ public final class LWCToolkit extends LWToolkit {
      */
     native boolean isApplicationActive();
 
+    /**
+     * Returns true if AWT toolkit is embedded, false otherwise.
+     *
+     * @return true if AWT toolkit is embedded, false otherwise
+     */
+    public static native boolean isEmbedded();
+
     /************************
      * Native methods section
      ************************/
diff --git a/src/macosx/native/sun/awt/LWCToolkit.m b/src/macosx/native/sun/awt/LWCToolkit.m
index 1d20e2333..1f1d1c414 100644
--- a/src/macosx/native/sun/awt/LWCToolkit.m
+++ b/src/macosx/native/sun/awt/LWCToolkit.m
@@ -452,3 +452,14 @@ Java_sun_font_FontManager_populateFontFileNameMap
 
 }
 
+/*
+ * Class:     sun_lwawt_macosx_LWCToolkit
+ * Method:    isEmbedded
+ * Signature: ()Z
+ */
+JNIEXPORT jboolean JNICALL
+Java_sun_lwawt_macosx_LWCToolkit_isEmbedded
+(JNIEnv *env, jclass klass) {
+    return [ThreadUtilities isAWTEmbedded] ? JNI_TRUE : JNI_FALSE;
+}
+