WifiStateMachine: onDown triggers wifi off

When the underlying sta interface in Client Mode gets an onDown
callback, notify SelfRecovery if MacRandomization isn't enabled.  We
do expect onDown to happen with MacRandomization, so we don't want to
restart/disable client mode...  because we would disable wifi and never connect.

This CL additionally adds a check to get the interface state after
creation.  This fixes the issue where WSM does not know the interface is
up before a connection.

This CL triggers SelfRecovery with a new reason code.  Also added unit
tests.

Due to b/72459123, supplicant reports of iface down are still handled.

Bug: 75989180
Bug: 75991970
Test: frameworks/opt/net/wifi/tests/wifitests/runtests.sh
Test: 'adb shell ifconfig wlan0 down' when connected, confirm wifi
disables
Test: 'adb shell ifconfig wlan0 down' when disconnected, confirm wifi
disables
Test: removed config while connected to the network does not disable
wifi
Test: removed config while disconnected does not disable wifi

Change-Id: Ifb7fc5289ed7f029ed02f6ce21d7b71710992501

3 files changed, 126 insertions, 43 deletions

diff --git a/service/java/com/android/server/wifi/SelfRecovery.java b/service/java/com/android/server/wifi/SelfRecovery.java
index d3985f5fd..a45de0466 100644
--- a/service/java/com/android/server/wifi/SelfRecovery.java
+++ b/service/java/com/android/server/wifi/SelfRecovery.java
@@ -38,11 +38,13 @@ public class SelfRecovery {
      */
     public static final int REASON_LAST_RESORT_WATCHDOG = 0;
     public static final int REASON_WIFINATIVE_FAILURE = 1;
+    public static final int REASON_STA_IFACE_DOWN = 2;
     public static final long MAX_RESTARTS_IN_TIME_WINDOW = 2; // 2 restarts per hour
     public static final long MAX_RESTARTS_TIME_WINDOW_MILLIS = 60 * 60 * 1000; // 1 hour
     protected static final String[] REASON_STRINGS = {
-            "Last Resort Watchdog", // REASON_LAST_RESORT_WATCHDOG
-            "WifiNative Failure"    // REASON_WIFINATIVE_FAILURE
+            "Last Resort Watchdog",  // REASON_LAST_RESORT_WATCHDOG
+            "WifiNative Failure",    // REASON_WIFINATIVE_FAILURE
+            "Sta Interface Down"     // REASON_STA_IFACE_DOWN
     };
 
     private final WifiController mWifiController;
@@ -59,16 +61,26 @@ public class SelfRecovery {
      * Trigger recovery.
      *
      * This method does the following:
-     * 1. Raises a wtf.
-     * 2. Sends {@link WifiController#CMD_RESTART_WIFI} to {@link WifiController} to initiate the
-     * stack restart.
+     * 1. Checks reason code used to trigger recovery
+     * 2. Checks for sta iface down triggers and disables wifi by sending {@link
+     * WifiController#CMD_RECOVERY_DISABLE_WIFI} to {@link WifiController} to disable wifi.
+     * 3. Throttles restart calls for underlying native failures
+     * 4. Sends {@link WifiController#CMD_RECOVERY_RESTART_WIFI} to {@link WifiController} to
+     * initiate the stack restart.
      * @param reason One of the above |REASON_*| codes.
      */
     public void trigger(int reason) {
-        if (!(reason == REASON_LAST_RESORT_WATCHDOG || reason == REASON_WIFINATIVE_FAILURE)) {
+        if (!(reason == REASON_LAST_RESORT_WATCHDOG || reason == REASON_WIFINATIVE_FAILURE
+                  || reason == REASON_STA_IFACE_DOWN)) {
             Log.e(TAG, "Invalid trigger reason. Ignoring...");
             return;
         }
+        if (reason == REASON_STA_IFACE_DOWN) {
+            Log.e(TAG, "STA interface down, disable wifi");
+            mWifiController.sendMessage(WifiController.CMD_RECOVERY_DISABLE_WIFI);
+            return;
+        }
+
         Log.e(TAG, "Triggering recovery for reason: " + REASON_STRINGS[reason]);
         if (reason == REASON_WIFINATIVE_FAILURE) {
             trimPastRestartTimes();
@@ -76,11 +88,12 @@ public class SelfRecovery {
             if (mPastRestartTimes.size() >= MAX_RESTARTS_IN_TIME_WINDOW) {
                 Log.e(TAG, "Already restarted wifi (" + MAX_RESTARTS_IN_TIME_WINDOW + ") times in"
                         + " last (" + MAX_RESTARTS_TIME_WINDOW_MILLIS + "ms ). Ignoring...");
+                // TODO: b/74093987  leaving wifi "broken" here, should disable wifi instead
                 return;
             }
             mPastRestartTimes.add(mClock.getElapsedSinceBootMillis());
         }
-        mWifiController.sendMessage(WifiController.CMD_RESTART_WIFI, reason);
+        mWifiController.sendMessage(WifiController.CMD_RECOVERY_RESTART_WIFI, reason);
     }
 
     /**
diff --git a/service/java/com/android/server/wifi/WifiController.java b/service/java/com/android/server/wifi/WifiController.java
index 5cc0306e1..3db804b9b 100644
--- a/service/java/com/android/server/wifi/WifiController.java
+++ b/service/java/com/android/server/wifi/WifiController.java
@@ -76,21 +76,23 @@ public class WifiController extends StateMachine {
 
     private static final int BASE = Protocol.BASE_WIFI_CONTROLLER;
 
-    static final int CMD_EMERGENCY_MODE_CHANGED        = BASE + 1;
-    static final int CMD_SCAN_ALWAYS_MODE_CHANGED      = BASE + 7;
-    static final int CMD_WIFI_TOGGLED                  = BASE + 8;
-    static final int CMD_AIRPLANE_TOGGLED              = BASE + 9;
-    static final int CMD_SET_AP                        = BASE + 10;
-    static final int CMD_DEFERRED_TOGGLE               = BASE + 11;
-    static final int CMD_USER_PRESENT                  = BASE + 12;
-    static final int CMD_AP_START_FAILURE              = BASE + 13;
-    static final int CMD_EMERGENCY_CALL_STATE_CHANGED  = BASE + 14;
-    static final int CMD_AP_STOPPED                    = BASE + 15;
-    static final int CMD_STA_START_FAILURE             = BASE + 16;
+    static final int CMD_EMERGENCY_MODE_CHANGED                 = BASE + 1;
+    static final int CMD_SCAN_ALWAYS_MODE_CHANGED               = BASE + 7;
+    static final int CMD_WIFI_TOGGLED                           = BASE + 8;
+    static final int CMD_AIRPLANE_TOGGLED                       = BASE + 9;
+    static final int CMD_SET_AP                                 = BASE + 10;
+    static final int CMD_DEFERRED_TOGGLE                        = BASE + 11;
+    static final int CMD_USER_PRESENT                           = BASE + 12;
+    static final int CMD_AP_START_FAILURE                       = BASE + 13;
+    static final int CMD_EMERGENCY_CALL_STATE_CHANGED           = BASE + 14;
+    static final int CMD_AP_STOPPED                             = BASE + 15;
+    static final int CMD_STA_START_FAILURE                      = BASE + 16;
     // Command used to trigger a wifi stack restart when in active mode
-    static final int CMD_RESTART_WIFI                  = BASE + 17;
+    static final int CMD_RECOVERY_RESTART_WIFI                  = BASE + 17;
     // Internal command used to complete wifi stack restart
-    private static final int CMD_RESTART_WIFI_CONTINUE = BASE + 18;
+    private static final int CMD_RECOVERY_RESTART_WIFI_CONTINUE = BASE + 18;
+    // Command to disable wifi when SelfRecovery is throttled or otherwise not doing full recovery
+    static final int CMD_RECOVERY_DISABLE_WIFI                  = BASE + 19;
 
     private DefaultState mDefaultState = new DefaultState();
     private StaEnabledState mStaEnabledState = new StaEnabledState();
@@ -215,8 +217,9 @@ public class WifiController extends StateMachine {
                 case CMD_AP_START_FAILURE:
                 case CMD_AP_STOPPED:
                 case CMD_STA_START_FAILURE:
-                case CMD_RESTART_WIFI:
-                case CMD_RESTART_WIFI_CONTINUE:
+                case CMD_RECOVERY_RESTART_WIFI:
+                case CMD_RECOVERY_RESTART_WIFI_CONTINUE:
+                case CMD_RECOVERY_DISABLE_WIFI:
                     break;
                 case CMD_USER_PRESENT:
                     mFirstUserSignOnSeen = true;
@@ -297,7 +300,7 @@ public class WifiController extends StateMachine {
                     log("DEFERRED_TOGGLE handled");
                     sendMessage((Message)(msg.obj));
                     break;
-                case CMD_RESTART_WIFI_CONTINUE:
+                case CMD_RECOVERY_RESTART_WIFI_CONTINUE:
                     transitionTo(mDeviceActiveState);
                     break;
                 default:
@@ -649,7 +652,7 @@ public class WifiController extends StateMachine {
                 }
                 mFirstUserSignOnSeen = true;
                 return HANDLED;
-            } else if (msg.what == CMD_RESTART_WIFI) {
+            } else if (msg.what == CMD_RECOVERY_RESTART_WIFI) {
                 final String bugTitle = "Wi-Fi BugReport";
                 final String bugDetail;
                 if (msg.obj != null && msg.arg1 < SelfRecovery.REASON_STRINGS.length
@@ -663,9 +666,12 @@ public class WifiController extends StateMachine {
                         mWifiStateMachine.takeBugReport(bugTitle, bugDetail);
                     });
                 }
-                deferMessage(obtainMessage(CMD_RESTART_WIFI_CONTINUE));
+                deferMessage(obtainMessage(CMD_RECOVERY_RESTART_WIFI_CONTINUE));
                 transitionTo(mApStaDisabledState);
                 return HANDLED;
+            } else if (msg.what == CMD_RECOVERY_DISABLE_WIFI) {
+                loge("Recovery has been throttled, disable wifi");
+                transitionTo(mApStaDisabledState);
             }
             return NOT_HANDLED;
         }
diff --git a/service/java/com/android/server/wifi/WifiStateMachine.java b/service/java/com/android/server/wifi/WifiStateMachine.java
index c96ea79fe..fc8ab84f2 100644
--- a/service/java/com/android/server/wifi/WifiStateMachine.java
+++ b/service/java/com/android/server/wifi/WifiStateMachine.java
@@ -230,15 +230,23 @@ public class WifiStateMachine extends StateMachine {
     private final InterfaceCallback mWifiNativeInterfaceCallback = new InterfaceCallback() {
         @Override
         public void onDestroyed(String ifaceName) {
-            sendMessage(CMD_INTERFACE_DESTROYED);
+            if (mInterfaceName != null && mInterfaceName.equals(ifaceName)) {
+                sendMessage(CMD_INTERFACE_DESTROYED);
+            }
         }
 
         @Override
         public void onUp(String ifaceName) {
+            if (mInterfaceName != null && mInterfaceName.equals(ifaceName)) {
+                sendMessage(CMD_INTERFACE_STATUS_CHANGED, 1);
+            }
         }
 
         @Override
         public void onDown(String ifaceName) {
+            if (mInterfaceName != null && mInterfaceName.equals(ifaceName)) {
+                sendMessage(CMD_INTERFACE_STATUS_CHANGED, 0);
+            }
         }
     };
     private boolean mIpReachabilityDisconnectEnabled = true;
@@ -460,12 +468,14 @@ public class WifiStateMachine extends StateMachine {
     static final int CMD_STOP_SUPPLICANT                                = BASE + 12;
     /* STA interface destroyed */
     static final int CMD_INTERFACE_DESTROYED                            = BASE + 13;
+    /* STA interface down */
+    static final int CMD_INTERFACE_DOWN                                 = BASE + 14;
     /* Indicates Static IP succeeded */
     static final int CMD_STATIC_IP_SUCCESS                              = BASE + 15;
     /* Indicates Static IP failed */
     static final int CMD_STATIC_IP_FAILURE                              = BASE + 16;
-    /* A delayed message sent to start driver when it fail to come up */
-    static final int CMD_DRIVER_START_TIMED_OUT                         = BASE + 19;
+    /* Interface status change */
+    static final int CMD_INTERFACE_STATUS_CHANGED                       = BASE + 20;
 
     /* Start the soft access point */
     static final int CMD_START_AP                                       = BASE + 21;
@@ -3390,6 +3400,25 @@ public class WifiStateMachine extends StateMachine {
                 + macRandomizationEnabled);
     }
 
+    /**
+     * Handle the error case where our underlying interface went down (if we do not have mac
+     * randomization enabled (b/72459123).
+     *
+     * This method triggers SelfRecovery with the error of REASON_STA_IFACE_DOWN.  SelfRecovery then
+     * decides if wifi should be restarted or disabled.
+     */
+    private void handleInterfaceDown() {
+        if (mEnableConnectedMacRandomization.get()) {
+            // interface will go down when mac randomization is active, skip
+            Log.d(TAG, "MacRandomization enabled, ignoring iface down");
+            return;
+        }
+
+        Log.e(TAG, "Detected an interface down, report failure to SelfRecovery");
+        // report a failure
+        mWifiInjector.getSelfRecovery().trigger(SelfRecovery.REASON_STA_IFACE_DOWN);
+    }
+
     /********************************************************
      * HSM states
      *******************************************************/
@@ -3498,7 +3527,6 @@ public class WifiStateMachine extends StateMachine {
                     break;
                 case CMD_START_SUPPLICANT:
                 case CMD_STOP_SUPPLICANT:
-                case CMD_DRIVER_START_TIMED_OUT:
                 case CMD_START_AP_FAILURE:
                 case CMD_STOP_AP:
                 case CMD_AP_STOPPED:
@@ -3534,6 +3562,8 @@ public class WifiStateMachine extends StateMachine {
                 case CMD_SELECT_TX_POWER_SCENARIO:
                 case CMD_WIFINATIVE_FAILURE:
                 case CMD_INTERFACE_DESTROYED:
+                case CMD_INTERFACE_DOWN:
+                case CMD_INTERFACE_STATUS_CHANGED:
                     messageHandlingStatus = MESSAGE_HANDLING_STATUS_DISCARD;
                     break;
                 case CMD_START_AP:
@@ -3722,6 +3752,23 @@ public class WifiStateMachine extends StateMachine {
     }
 
     class InitialState extends State {
+        private boolean mIfaceIsUp;
+
+        private void onUpChanged(boolean isUp) {
+            if (isUp == mIfaceIsUp) {
+                return;  // no change
+            }
+            mIfaceIsUp = isUp;
+            if (isUp) {
+                Log.d(TAG, "Client mode interface is up");
+                // for now, do nothing - client mode has never waited for iface up
+            } else {
+                // A driver/firmware hang can now put the interface in a down state.
+                // We detect the interface going down and recover from it
+                handleInterfaceDown();
+            }
+        }
+
         private void cleanup() {
             // tell scanning service that scans are not available - about to kill the interface and
             // supplicant
@@ -3737,10 +3784,12 @@ public class WifiStateMachine extends StateMachine {
             // TODO: Remove this big hammer. We cannot support concurrent interfaces with this!
             mWifiNative.teardownAllInterfaces();
             mInterfaceName = null;
+            mIfaceIsUp = false;
         }
 
         @Override
         public void enter() {
+            mIfaceIsUp = false;
             mWifiMonitor.stopAllMonitoring();
             mWifiStateTracker.updateState(WifiStateTracker.INVALID);
             cleanup();
@@ -3761,6 +3810,9 @@ public class WifiStateMachine extends StateMachine {
                         transitionTo(mDefaultState);
                         break;
                     }
+                    // now that we have the interface, initialize our up/down status
+                    onUpChanged(mWifiNative.isInterfaceUp(mInterfaceName));
+
                     mIpClient = mFacade.makeIpClient(
                             mContext, mInterfaceName, new IpClientCallback());
                     mIpClient.setMulticastFilter(true);
@@ -3783,6 +3835,20 @@ public class WifiStateMachine extends StateMachine {
                             WifiDiagnostics.REPORT_REASON_WIFINATIVE_FAILURE);
                     mWifiInjector.getSelfRecovery().trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
                     break;
+                case CMD_INTERFACE_STATUS_CHANGED:
+                    boolean isUp = message.arg1 == 1;
+                    // For now, this message can be triggered due to link state and/or interface
+                    // status changes (b/77218676).  First check if we really see an iface down by
+                    // consulting our view of supplicant state.
+                    if (!isUp && SupplicantState.isDriverActive(mWifiInfo.getSupplicantState())) {
+                        // the driver is active, so this could just be part of normal operation, do
+                        // not disable wifi in these cases (ex, a network was removed) or worry
+                        // about the link status
+                        break;
+                    }
+
+                    onUpChanged(isUp);
+                    break;
                 default:
                     return NOT_HANDLED;
             }
@@ -4343,20 +4409,6 @@ public class WifiStateMachine extends StateMachine {
                     break;
                 case WifiMonitor.SUPPLICANT_STATE_CHANGE_EVENT:
                     SupplicantState state = handleSupplicantStateChange(message);
-                    // A driver/firmware hang can now put the interface in a down state.
-                    // We detect the interface going down and recover from it
-                    if (!SupplicantState.isDriverActive(state) && !mModeChange
-                            && !mEnableConnectedMacRandomization.get()) {
-                        if (mNetworkInfo.getState() != NetworkInfo.State.DISCONNECTED) {
-                            handleNetworkDisconnect();
-                        }
-                        log("Detected an interface down, restart driver");
-                        // Rely on the fact that this will force us into killing supplicant and then
-                        // restart supplicant from a clean state.
-                        sendMessage(CMD_START_SUPPLICANT);
-                        transitionTo(mInitialState);
-                        break;
-                    }
 
                     // Supplicant can fail to report a NETWORK_DISCONNECTION_EVENT
                     // when authentication times out after a successful connection,
@@ -4378,6 +4430,18 @@ public class WifiStateMachine extends StateMachine {
                     if (state == SupplicantState.COMPLETED) {
                         mIpClient.confirmConfiguration();
                     }
+
+                    if (!SupplicantState.isDriverActive(state)) {
+                        // still use supplicant to detect interface down while work to
+                        // mitigate b/77218676 is in progress
+                        // note: explicitly using this command to dedup iface down notification
+                        // paths (onUpChanged filters out duplicate updates)
+                        sendMessage(CMD_INTERFACE_STATUS_CHANGED, 0);
+                        if (mVerboseLoggingEnabled) {
+                            Log.d(TAG, "detected interface down via supplicant");
+                        }
+                    }
+
                     break;
                 case WifiP2pServiceImpl.DISCONNECT_WIFI_REQUEST:
                     if (message.arg1 == 1) {