WifiStateMachinePrime: centralize native failure handling

Instead of having each mode listen and react to underlying native daemon
failures, centralize the control in WifiStateMachinePrime.  Now all
modes have the benefit of detection and SelfRecovery (when applicable).

Note: Multiple notifications for underlying failures is interfering with
recovery, tested with wifi hal death that only sends one callback.
Supplicant and wificond failures still trigger multiple callbacks and
recovery needs to be verified once those are fixed.

Bug: 75989180
Bug: 31346104
Test: frameworks/opt/net/wifi/tests/wifitests/runtests.sh
Test: triggered wifi hal failures with 'kill' while in all modes, wifi
re-entered scan and client modes as appropriate.

Change-Id: I2b606b25e463b1f5e152cf6a6ff2a188e5c48d6d

6 files changed, 47 insertions, 219 deletions

diff --git a/tests/wifitests/src/com/android/server/wifi/ClientModeManagerTest.java b/tests/wifitests/src/com/android/server/wifi/ClientModeManagerTest.java
index 942ff1c8b..e48988c3c 100644
--- a/tests/wifitests/src/com/android/server/wifi/ClientModeManagerTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/ClientModeManagerTest.java
@@ -65,8 +65,6 @@ public class ClientModeManagerTest {
     @Mock WifiMonitor mWifiMonitor;
     @Mock ScanRequestProxy mScanRequestProxy;
 
-    final ArgumentCaptor<WifiNative.StatusListener> mStatusListenerCaptor =
-            ArgumentCaptor.forClass(WifiNative.StatusListener.class);
     final ArgumentCaptor<WifiNative.InterfaceCallback> mInterfaceCallbackCaptor =
             ArgumentCaptor.forClass(WifiNative.InterfaceCallback.class);
 
@@ -92,7 +90,6 @@ public class ClientModeManagerTest {
         mClientModeManager.start();
         mLooper.dispatchAll();
 
-        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
         verify(mWifiNative).setupInterfaceForClientMode(
                 eq(false), mInterfaceCallbackCaptor.capture());
 
@@ -157,7 +154,7 @@ public class ClientModeManagerTest {
 
         List<Intent> intents = intentCaptor.getAllValues();
         assertEquals(3, intents.size());
-        checkWifiStateChangedBroadcast(intents.get(0), WIFI_STATE_DISABLING, WIFI_STATE_ENABLED);
+        checkWifiStateChangedBroadcast(intents.get(0), WIFI_STATE_DISABLING, WIFI_STATE_UNKNOWN);
         checkWifiScanStateChangedBroadcast(intents.get(1), WIFI_STATE_DISABLED);
         checkWifiStateChangedBroadcast(intents.get(2), WIFI_STATE_DISABLED, WIFI_STATE_DISABLING);
         checkWifiStateChangeListenerUpdate(WIFI_STATE_DISABLED);
@@ -267,40 +264,6 @@ public class ClientModeManagerTest {
     }
 
     /**
-     * Testing the handling of a wifinative failure status change notification.
-     */
-    @Test
-    public void clientModeStartedStopsOnNativeFailure() throws Exception {
-        startClientModeAndVerifyEnabled();
-        reset(mContext, mScanRequestProxy, mListener);
-        mStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchNext();
-
-        checkWifiStateChangeListenerUpdate(WIFI_STATE_UNKNOWN);
-
-        mLooper.dispatchAll();
-
-        verifyNotificationsForFailure();
-    }
-
-    /**
-     * Testing that handling of a wifinative callback that is not a failuer does not stop client
-     * mode.
-     */
-    @Test
-    public void clientModeStartedAndStaysUpOnNativeNonFailureCallback() throws Exception {
-        startClientModeAndVerifyEnabled();
-        reset(mContext, mScanRequestProxy, mListener);
-        mStatusListenerCaptor.getValue().onStatusChanged(true);
-        mLooper.dispatchAll();
-
-        verify(mListener, never()).onStateChanged(eq(WIFI_STATE_UNKNOWN));
-        verify(mListener, never()).onStateChanged(eq(WIFI_STATE_DISABLING));
-        verify(mListener, never()).onStateChanged(eq(WIFI_STATE_DISABLED));
-    }
-
-
-    /**
      * Testing the handling of an interface destroyed notification.
      */
     @Test
diff --git a/tests/wifitests/src/com/android/server/wifi/ScanOnlyModeManagerTest.java b/tests/wifitests/src/com/android/server/wifi/ScanOnlyModeManagerTest.java
index 2556dfcf4..1b4095bd0 100644
--- a/tests/wifitests/src/com/android/server/wifi/ScanOnlyModeManagerTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/ScanOnlyModeManagerTest.java
@@ -69,8 +69,6 @@ public class ScanOnlyModeManagerTest {
     @Mock ScanRequestProxy mScanRequestProxy;
     @Mock WakeupController mWakeupController;
 
-    final ArgumentCaptor<WifiNative.StatusListener> mStatusListenerCaptor =
-            ArgumentCaptor.forClass(WifiNative.StatusListener.class);
     final ArgumentCaptor<WifiNative.InterfaceCallback> mInterfaceCallbackCaptor =
             ArgumentCaptor.forClass(WifiNative.InterfaceCallback.class);
 
@@ -94,7 +92,6 @@ public class ScanOnlyModeManagerTest {
         mScanOnlyModeManager.start();
         mLooper.dispatchAll();
 
-        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
         verify(mWifiNative).setupInterfaceForClientMode(eq(true),
                 mInterfaceCallbackCaptor.capture());
 
@@ -131,37 +128,6 @@ public class ScanOnlyModeManagerTest {
     }
 
     /**
-     * ScanMode idle state does not crash when a native status update comes before entering the
-     * active state.
-     */
-    @Test
-    public void scanModeNativeUpdateBeforeStartDoesNotCrash() throws Exception {
-        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
-
-        mStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-        verifyNoMoreInteractions(mContext, mListener);
-
-        verify(mScanRequestProxy, never()).clearScanResults();
-    }
-
-    /**
-     * ScanMode increments failure metrics when failing to setup client mode.
-     */
-    @Test
-    public void detectAndReportErrorWhenSetupForClientWifiNativeFailure() throws Exception {
-        when(mWifiNative.setupInterfaceForClientMode(anyBoolean(), any())).thenReturn(null);
-        mScanOnlyModeManager.start();
-        mLooper.dispatchAll();
-
-        ArgumentCaptor<Intent> intentCaptor = ArgumentCaptor.forClass(Intent.class);
-        verify(mContext, atLeastOnce()).sendStickyBroadcastAsUser(intentCaptor.capture(),
-                eq(UserHandle.ALL));
-        checkWifiScanStateChangedBroadcast(intentCaptor.getValue(), WIFI_STATE_DISABLED);
-        checkWifiStateChangeListenerUpdate(WIFI_STATE_UNKNOWN);
-    }
-
-    /**
      * ScanMode start does not indicate scanning is available when the interface name is empty.
      */
     @Test
@@ -278,41 +244,6 @@ public class ScanOnlyModeManagerTest {
         verify(mScanRequestProxy, never()).clearScanResults();
     }
 
-
-    /**
-     * Testing the handling of a WifiNative failure status change notification.
-     */
-    @Test
-    public void scanModeStartedStopsOnNativeFailure() throws Exception {
-        startScanOnlyModeAndVerifyEnabled();
-        reset(mContext);
-        mStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-        ArgumentCaptor<Intent> intentCaptor = ArgumentCaptor.forClass(Intent.class);
-        verify(mContext).sendStickyBroadcastAsUser(intentCaptor.capture(),
-                                                   eq(UserHandle.ALL));
-
-        checkWifiScanStateChangedBroadcast(intentCaptor.getValue(), WIFI_STATE_DISABLED);
-        checkWifiStateChangeListenerUpdate(WIFI_STATE_UNKNOWN);
-        checkWifiStateChangeListenerUpdate(WIFI_STATE_DISABLED);
-        verify(mScanRequestProxy).clearScanResults();
-    }
-
-    /**
-     * WifiNative callback that does not indicate failure should not stop Scan mode.
-     */
-    @Test
-    public void scanModeStartedDoesNotStopOnNativeSuccessUpdate() throws Exception {
-        startScanOnlyModeAndVerifyEnabled();
-        reset(mContext, mListener);
-        mStatusListenerCaptor.getValue().onStatusChanged(true);
-        mLooper.dispatchAll();
-
-        verifyNoMoreInteractions(mContext, mListener);
-
-        verify(mScanRequestProxy, never()).clearScanResults();
-    }
-
     /**
      * Entering StartedState starts the WakeupController.
      */
diff --git a/tests/wifitests/src/com/android/server/wifi/SoftApManagerTest.java b/tests/wifitests/src/com/android/server/wifi/SoftApManagerTest.java
index e41af6867..bce5e8e94 100644
--- a/tests/wifitests/src/com/android/server/wifi/SoftApManagerTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/SoftApManagerTest.java
@@ -93,8 +93,6 @@ public class SoftApManagerTest {
     @Mock FrameworkFacade mFrameworkFacade;
     @Mock WifiApConfigStore mWifiApConfigStore;
     @Mock WifiMetrics mWifiMetrics;
-    final ArgumentCaptor<WifiNative.StatusListener> mWifiNativeStatusListenerCaptor =
-            ArgumentCaptor.forClass(WifiNative.StatusListener.class);
     final ArgumentCaptor<WifiNative.InterfaceCallback> mWifiNativeInterfaceCallbackCaptor =
             ArgumentCaptor.forClass(WifiNative.InterfaceCallback.class);
     final ArgumentCaptor<WifiNative.SoftApListener> mSoftApListenerCaptor =
@@ -140,7 +138,6 @@ public class SoftApManagerTest {
                                                            config,
                                                            mWifiMetrics);
         mLooper.dispatchAll();
-        verify(mWifiNative).registerStatusListener(mWifiNativeStatusListenerCaptor.capture());
 
         return newSoftApManager;
     }
@@ -473,71 +470,6 @@ public class SoftApManagerTest {
     }
 
     /**
-     * Verify that SoftAp mode shuts down on wifinative failure.
-     */
-    @Test
-    public void handlesWifiNativeFailure() throws Exception {
-        SoftApModeConfiguration softApModeConfig =
-                new SoftApModeConfiguration(WifiManager.IFACE_IP_MODE_TETHERED, null);
-        startSoftApAndVerifyEnabled(softApModeConfig);
-
-        // reset to clear verified Intents for ap state change updates
-        reset(mContext);
-
-        mWifiNativeStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-        InOrder order = inOrder(mCallback);
-        order.verify(mCallback).onStateChanged(WifiManager.WIFI_AP_STATE_FAILED,
-                WifiManager.SAP_START_FAILURE_GENERAL);
-        ArgumentCaptor<Intent> intentCaptor = ArgumentCaptor.forClass(Intent.class);
-        verify(mContext, times(3)).sendStickyBroadcastAsUser(intentCaptor.capture(),
-                eq(UserHandle.ALL));
-
-        List<Intent> capturedIntents = intentCaptor.getAllValues();
-        checkApStateChangedBroadcast(capturedIntents.get(0), WIFI_AP_STATE_FAILED,
-                WIFI_AP_STATE_ENABLED, WifiManager.SAP_START_FAILURE_GENERAL, TEST_INTERFACE_NAME,
-                softApModeConfig.getTargetMode());
-        checkApStateChangedBroadcast(capturedIntents.get(1), WIFI_AP_STATE_DISABLING,
-                WIFI_AP_STATE_FAILED, HOTSPOT_NO_ERROR, TEST_INTERFACE_NAME,
-                softApModeConfig.getTargetMode());
-        checkApStateChangedBroadcast(capturedIntents.get(2), WIFI_AP_STATE_DISABLED,
-                WIFI_AP_STATE_DISABLING, HOTSPOT_NO_ERROR, TEST_INTERFACE_NAME,
-                softApModeConfig.getTargetMode());
-    }
-
-    /**
-     * Verify that SoftAp does not crash on wifinative failure before it is started.
-     */
-    @Test
-    public void handlesWifiNativeFailureBeforeStart() throws Exception {
-        SoftApModeConfiguration softApModeConfig =
-                new SoftApModeConfiguration(WifiManager.IFACE_IP_MODE_TETHERED, null);
-
-        mSoftApManager = createSoftApManager(softApModeConfig);
-
-        mWifiNativeStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-    }
-
-    /**
-     * Verify that SoftAp mode does not shut down on wifinative success update.
-     */
-    @Test
-    public void handlesWifiNativeSuccess() throws Exception {
-        SoftApModeConfiguration softApModeConfig =
-                new SoftApModeConfiguration(WifiManager.IFACE_IP_MODE_TETHERED, null);
-        startSoftApAndVerifyEnabled(softApModeConfig);
-
-        // reset to clear verified Intents for ap state change updates
-        reset(mContext, mCallback);
-
-        mWifiNativeStatusListenerCaptor.getValue().onStatusChanged(true);
-        mLooper.dispatchAll();
-
-        verifyNoMoreInteractions(mContext, mCallback);
-    }
-
-    /**
      * Verify that onDestroyed properly reports softap stop.
      */
     @Test
diff --git a/tests/wifitests/src/com/android/server/wifi/WifiControllerTest.java b/tests/wifitests/src/com/android/server/wifi/WifiControllerTest.java
index 279fb670b..5211964a6 100644
--- a/tests/wifitests/src/com/android/server/wifi/WifiControllerTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/WifiControllerTest.java
@@ -466,9 +466,11 @@ public class WifiControllerTest {
 
         reset(mWifiStateMachine);
         assertEquals("ApStaDisabledState", getCurrentState().getName());
+
+        reset(mWifiStateMachinePrime);
         mWifiController.sendMessage(CMD_RECOVERY_RESTART_WIFI);
         mLooper.dispatchAll();
-        verifyZeroInteractions(mWifiStateMachine);
+        verify(mWifiStateMachinePrime).disableWifi();
     }
 
     /**
@@ -484,9 +486,12 @@ public class WifiControllerTest {
     public void testRestartWifiStackInStaDisabledWithScanState() throws Exception {
         reset(mWifiStateMachine);
         assertEquals("StaDisabledWithScanState", getCurrentState().getName());
+        reset(mWifiStateMachinePrime);
         mWifiController.sendMessage(CMD_RECOVERY_RESTART_WIFI);
         mLooper.dispatchAll();
-        verifyZeroInteractions(mWifiStateMachine);
+        InOrder inOrder = inOrder(mWifiStateMachinePrime);
+        verify(mWifiStateMachinePrime).disableWifi();
+        verify(mWifiStateMachinePrime).enterScanOnlyMode();
     }
 
     /**
@@ -550,10 +555,9 @@ public class WifiControllerTest {
         verify(mWifiStateMachinePrime).enterSoftAPMode(any());
         assertEquals("ApEnabledState", getCurrentState().getName());
 
-        reset(mWifiStateMachine);
+        reset(mWifiStateMachinePrime);
         mWifiController.sendMessage(CMD_RECOVERY_RESTART_WIFI);
         mLooper.dispatchAll();
-        verifyZeroInteractions(mWifiStateMachine);
-        verify(mWifiStateMachinePrime, never()).disableWifi();
+        verify(mWifiStateMachinePrime).disableWifi();
     }
 }
diff --git a/tests/wifitests/src/com/android/server/wifi/WifiStateMachinePrimeTest.java b/tests/wifitests/src/com/android/server/wifi/WifiStateMachinePrimeTest.java
index ae8d8cd95..9deb628eb 100644
--- a/tests/wifitests/src/com/android/server/wifi/WifiStateMachinePrimeTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/WifiStateMachinePrimeTest.java
@@ -30,6 +30,7 @@ import com.android.internal.app.IBatteryStats;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 import org.mockito.invocation.InvocationOnMock;
@@ -56,11 +57,17 @@ public class WifiStateMachinePrimeTest {
     @Mock SoftApManager mSoftApManager;
     @Mock DefaultModeManager mDefaultModeManager;
     @Mock IBatteryStats mBatteryStats;
+    @Mock SelfRecovery mSelfRecovery;
+    @Mock BaseWifiDiagnostics mWifiDiagnostics;
     ScanOnlyModeManager.Listener mScanOnlyListener;
     WifiManager.SoftApCallback mSoftApManagerCallback;
     @Mock WifiManager.SoftApCallback mSoftApStateMachineCallback;
+    WifiNative.StatusListener mWifiNativeStatusListener;
     WifiStateMachinePrime mWifiStateMachinePrime;
 
+    final ArgumentCaptor<WifiNative.StatusListener> mStatusListenerCaptor =
+            ArgumentCaptor.forClass(WifiNative.StatusListener.class);
+
     /**
      * Set up the test environment.
      */
@@ -71,9 +78,15 @@ public class WifiStateMachinePrimeTest {
         MockitoAnnotations.initMocks(this);
         mLooper = new TestLooper();
 
+        when(mWifiInjector.getSelfRecovery()).thenReturn(mSelfRecovery);
+        when(mWifiInjector.makeWifiDiagnostics(eq(mWifiNative))).thenReturn(mWifiDiagnostics);
+
         mWifiStateMachinePrime = createWifiStateMachinePrime();
         mLooper.dispatchAll();
 
+        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
+        mWifiNativeStatusListener = mStatusListenerCaptor.getValue();
+
         mWifiStateMachinePrime.registerSoftApCallback(mSoftApStateMachineCallback);
     }
 
@@ -539,4 +552,28 @@ public class WifiStateMachinePrimeTest {
         // since we start up in disabled, this should not re-enter the disabled state
         verify(mWifiNative).teardownAllInterfaces();
     }
+
+    /**
+     * Trigger recovery and a bug report if we see a native failure.
+     */
+    @Test
+    public void handleWifiNativeFailure() throws Exception {
+        mWifiNativeStatusListener.onStatusChanged(false);
+        mLooper.dispatchAll();
+        verify(mWifiDiagnostics).captureBugReportData(
+                WifiDiagnostics.REPORT_REASON_WIFINATIVE_FAILURE);
+        verify(mSelfRecovery).trigger(eq(SelfRecovery.REASON_WIFINATIVE_FAILURE));
+    }
+
+    /**
+     * Verify an onStatusChanged callback with "true" does not trigger recovery.
+     */
+    @Test
+    public void handleWifiNativeStatusReady() throws Exception {
+        mWifiNativeStatusListener.onStatusChanged(true);
+        mLooper.dispatchAll();
+        verify(mWifiDiagnostics, never()).captureBugReportData(
+                WifiDiagnostics.REPORT_REASON_WIFINATIVE_FAILURE);
+        verify(mSelfRecovery, never()).trigger(eq(SelfRecovery.REASON_WIFINATIVE_FAILURE));
+    }
 }
diff --git a/tests/wifitests/src/com/android/server/wifi/WifiStateMachineTest.java b/tests/wifitests/src/com/android/server/wifi/WifiStateMachineTest.java
index c90bde3bb..28353074d 100644
--- a/tests/wifitests/src/com/android/server/wifi/WifiStateMachineTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/WifiStateMachineTest.java
@@ -371,8 +371,6 @@ public class WifiStateMachineTest {
     @Mock WakeupController mWakeupController;
     @Mock ScanRequestProxy mScanRequestProxy;
 
-    final ArgumentCaptor<WifiNative.StatusListener> mStatusListenerCaptor =
-            ArgumentCaptor.forClass(WifiNative.StatusListener.class);
     final ArgumentCaptor<WifiNative.InterfaceCallback> mInterfaceCallbackCaptor =
             ArgumentCaptor.forClass(WifiNative.InterfaceCallback.class);
 
@@ -1783,43 +1781,6 @@ public class WifiStateMachineTest {
     }
 
     /**
-     * Trigger recovery and a bug report of we see a native failure when Client mode is active.
-     */
-    @Test
-    public void handleWifiNativeFailureWhenClientModeActive() throws Exception {
-        // Trigger initialize to capture the death handler registration.
-        loadComponentsInStaMode();
-
-        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
-
-        // Now trigger the death notification.
-        mStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-        verify(mSelfRecovery).trigger(eq(SelfRecovery.REASON_WIFINATIVE_FAILURE));
-        verify(mWifiDiagnostics).captureBugReportData(
-                WifiDiagnostics.REPORT_REASON_WIFINATIVE_FAILURE);
-    }
-
-    /**
-     * WifiNative failures when in a state other than client mode should just be dropped.
-     */
-    @Test
-    public void handleWifiNativeFailureInDefaultDoesNotRestartClientMode() throws Exception {
-        // Trigger initialize to capture the death handler registration.
-        loadComponentsInStaMode();
-        verify(mWifiNative).registerStatusListener(mStatusListenerCaptor.capture());
-
-        mWsm.setOperationalMode(WifiStateMachine.SCAN_ONLY_MODE);
-        mLooper.dispatchAll();
-
-        // Now trigger the death notification.
-        mStatusListenerCaptor.getValue().onStatusChanged(false);
-        mLooper.dispatchAll();
-        verify(mSelfRecovery, never()).trigger(anyInt());
-        verify(mWifiDiagnostics, never()).captureBugReportData(anyInt());
-    }
-
-    /**
      * Test verifying that interface onDown callback triggers SelfRecovery when Supplicant has
      * already reported the driver is not active.
      */