SelfRecovery: Add overlay for the recovery limit

Bug: 153395624
Test: atest com.android.server.wifi
Test: Manually crash wpa_supplicant from shell:
- Ensured that we restarted the stack the first 2 attempts.
- Ensured that we disabled wifi for all further attempts.

Change-Id: Iafcd193e373f7e6530446e69a927e2dd0056f402

5 files changed, 71 insertions, 32 deletions

diff --git a/service/java/com/android/server/wifi/SelfRecovery.java b/service/java/com/android/server/wifi/SelfRecovery.java
index e789dd3a4..b7eb72364 100644
--- a/service/java/com/android/server/wifi/SelfRecovery.java
+++ b/service/java/com/android/server/wifi/SelfRecovery.java
@@ -17,12 +17,16 @@
 package com.android.server.wifi;
 
 import android.annotation.IntDef;
+import android.content.Context;
 import android.util.Log;
 
+import com.android.wifi.resources.R;
+
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.util.Iterator;
 import java.util.LinkedList;
+import java.util.concurrent.TimeUnit;
 
 /**
  * This class is used to recover the wifi stack from a fatal failure. The recovery mechanism
@@ -50,19 +54,19 @@ public class SelfRecovery {
             REASON_STA_IFACE_DOWN})
     public @interface RecoveryReason {}
 
-    public static final long MAX_RESTARTS_IN_TIME_WINDOW = 2; // 2 restarts per hour
-    public static final long MAX_RESTARTS_TIME_WINDOW_MILLIS = 60 * 60 * 1000; // 1 hour
     protected static final String[] REASON_STRINGS = {
             "Last Resort Watchdog",  // REASON_LAST_RESORT_WATCHDOG
             "WifiNative Failure",    // REASON_WIFINATIVE_FAILURE
             "Sta Interface Down"     // REASON_STA_IFACE_DOWN
     };
 
+    private final Context mContext;
     private final ActiveModeWarden mActiveModeWarden;
     private final Clock mClock;
     // Time since boot (in millis) that restart occurred
     private final LinkedList<Long> mPastRestartTimes;
-    public SelfRecovery(ActiveModeWarden activeModeWarden, Clock clock) {
+    public SelfRecovery(Context context, ActiveModeWarden activeModeWarden, Clock clock) {
+        mContext = context;
         mActiveModeWarden = activeModeWarden;
         mClock = clock;
         mPastRestartTimes = new LinkedList<>();
@@ -94,11 +98,17 @@ public class SelfRecovery {
 
         Log.e(TAG, "Triggering recovery for reason: " + REASON_STRINGS[reason]);
         if (reason == REASON_WIFINATIVE_FAILURE) {
+            int maxRecoveriesPerHour = mContext.getResources().getInteger(
+                    R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour);
+            if (maxRecoveriesPerHour == 0) {
+                Log.e(TAG, "Recovery disabled. Disabling wifi");
+                mActiveModeWarden.recoveryDisableWifi();
+                return;
+            }
             trimPastRestartTimes();
-            // Ensure there haven't been too many restarts within MAX_RESTARTS_TIME_WINDOW
-            if (mPastRestartTimes.size() >= MAX_RESTARTS_IN_TIME_WINDOW) {
-                Log.e(TAG, "Already restarted wifi (" + MAX_RESTARTS_IN_TIME_WINDOW + ") times in"
-                        + " last (" + MAX_RESTARTS_TIME_WINDOW_MILLIS + "ms ). Disabling wifi");
+            if (mPastRestartTimes.size() >= maxRecoveriesPerHour) {
+                Log.e(TAG, "Already restarted wifi " + maxRecoveriesPerHour + " times in"
+                        + " last 1 hour. Disabling wifi");
                 mActiveModeWarden.recoveryDisableWifi();
                 return;
             }
@@ -115,7 +125,7 @@ public class SelfRecovery {
         long now = mClock.getElapsedSinceBootMillis();
         while (iter.hasNext()) {
             Long restartTimeMillis = iter.next();
-            if (now - restartTimeMillis > MAX_RESTARTS_TIME_WINDOW_MILLIS) {
+            if (now - restartTimeMillis > TimeUnit.HOURS.toMillis(1)) {
                 iter.remove();
             } else {
                 break;
diff --git a/service/java/com/android/server/wifi/WifiInjector.java b/service/java/com/android/server/wifi/WifiInjector.java
index 594008975..1e5e54682 100644
--- a/service/java/com/android/server/wifi/WifiInjector.java
+++ b/service/java/com/android/server/wifi/WifiInjector.java
@@ -369,7 +369,7 @@ public class WifiInjector {
                 this, mFrameworkFacade, mClock);
         mLockManager = new WifiLockManager(mContext, mBatteryStats,
                 mClientModeImpl, mFrameworkFacade, wifiHandler, mWifiNative, mClock, mWifiMetrics);
-        mSelfRecovery = new SelfRecovery(mActiveModeWarden, mClock);
+        mSelfRecovery = new SelfRecovery(mContext, mActiveModeWarden, mClock);
         mWifiMulticastLockManager = new WifiMulticastLockManager(
                 mClientModeImpl.getMcastLockManagerFilterController(), mBatteryStats);
         mDppManager = new DppManager(wifiHandler, mWifiNative,
diff --git a/service/res/values/config.xml b/service/res/values/config.xml
index 4d351b105..43d4a03c5 100644
--- a/service/res/values/config.xml
+++ b/service/res/values/config.xml
@@ -399,4 +399,9 @@
 
     <!-- Enable WPA2 to WPA3 auto-upgrade offload to capable Driver/Firmware -->
     <bool translatable="false" name="config_wifiSaeUpgradeOffloadEnabled">false</bool>
+
+    <!-- Number of self recoveries to be attempted per hour. Any fatal errors beyond this will
+         cause the wifi stack to turn wifi off and wait for user input.
+         Set to 0 to turn off recovery attempts and always turn off wifi on failures -->
+    <integer translatable="false" name="config_wifiMaxNativeFailureSelfRecoveryPerHour">2</integer>
 </resources>
diff --git a/service/res/values/overlayable.xml b/service/res/values/overlayable.xml
index 7244911af..028d7240f 100644
--- a/service/res/values/overlayable.xml
+++ b/service/res/values/overlayable.xml
@@ -126,6 +126,7 @@
           <item type="integer" name="config_wifiPollRssiIntervalMilliseconds" />
           <item type="bool" name="config_wifiSaeUpgradeEnabled" />
           <item type="bool" name="config_wifiSaeUpgradeOffloadEnabled" />
+          <item type="integer" name="config_wifiMaxNativeFailureSelfRecoveryPerHour" />
           <!-- Params from config.xml that can be overlayed -->
 
           <!-- Params from strings.xml that can be overlayed -->
diff --git a/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java b/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
index fe8bfd575..3538598c3 100644
--- a/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
@@ -19,25 +19,39 @@ package com.android.server.wifi;
 import static org.mockito.Mockito.*;
 import static org.mockito.MockitoAnnotations.*;
 
+import android.content.Context;
+
 import androidx.test.filters.SmallTest;
 
+import com.android.wifi.resources.R;
+
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mock;
 
+import java.util.concurrent.TimeUnit;
+
 /**
  * Unit tests for {@link com.android.server.wifi.SelfRecovery}.
  */
 @SmallTest
 public class SelfRecoveryTest extends WifiBaseTest {
+    private static final int DEFAULT_MAX_RECOVERY_PER_HOUR = 2;
     SelfRecovery mSelfRecovery;
+    MockResources mResources;
+    @Mock Context mContext;
     @Mock ActiveModeWarden mActiveModeWarden;
     @Mock Clock mClock;
 
     @Before
     public void setUp() throws Exception {
         initMocks(this);
-        mSelfRecovery = new SelfRecovery(mActiveModeWarden, mClock);
+        mResources = new MockResources();
+        // Default value of 2 recovery per hour.
+        mResources.setInteger(R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour,
+                DEFAULT_MAX_RECOVERY_PER_HOUR);
+        when(mContext.getResources()).thenReturn(mResources);
+        mSelfRecovery = new SelfRecovery(mContext, mActiveModeWarden, mClock);
     }
 
     /**
@@ -51,7 +65,7 @@ public class SelfRecoveryTest extends WifiBaseTest {
         reset(mActiveModeWarden);
 
         when(mClock.getElapsedSinceBootMillis())
-                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
+                .thenReturn(TimeUnit.HOURS.toMillis(1) + 1);
         mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
         verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
         reset(mActiveModeWarden);
@@ -80,25 +94,16 @@ public class SelfRecoveryTest extends WifiBaseTest {
     }
 
     /**
-     * Verifies that invocations of {@link SelfRecovery#trigger(int)} for REASON_HAL_CRASH &
-     * REASON_WIFICOND_CRASH are limited to {@link SelfRecovery#MAX_RESTARTS_IN_TIME_WINDOW} in a
-     * {@link SelfRecovery#MAX_RESTARTS_TIME_WINDOW_MILLIS} millisecond time window.
+     * Verifies that invocations of {@link SelfRecovery#trigger(int)} for REASON_WIFI_NATIVE
+     * are limited to {@link R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour} in a
+     * 1 hour time window.
      */
     @Test
     public void testTimeWindowLimiting_typicalUse() {
         when(mClock.getElapsedSinceBootMillis()).thenReturn(0L);
         // Fill up the SelfRecovery's restart time window buffer, ensure all the restart triggers
         // aren't ignored
-        for (int i = 0; i < SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW / 2; i++) {
-            mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
-            verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
-            reset(mActiveModeWarden);
-
-            mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
-            verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
-            reset(mActiveModeWarden);
-        }
-        if ((SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW % 2) == 1) {
+        for (int i = 0; i < DEFAULT_MAX_RECOVERY_PER_HOUR; i++) {
             mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
             verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
             reset(mActiveModeWarden);
@@ -128,27 +133,46 @@ public class SelfRecoveryTest extends WifiBaseTest {
 
         // now TRAVEL FORWARDS IN TIME and ensure that more restarts can occur
         when(mClock.getElapsedSinceBootMillis())
-                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
+                .thenReturn(TimeUnit.HOURS.toMillis(1) + 1);
         mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
         verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
         reset(mActiveModeWarden);
 
         when(mClock.getElapsedSinceBootMillis())
-                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
+                .thenReturn(TimeUnit.HOURS.toMillis(1) + 1);
         mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
         verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
         reset(mActiveModeWarden);
     }
 
     /**
+     * Verifies that invocations of {@link SelfRecovery#trigger(int)} for REASON_WIFI_NATIVE
+     * does not trigger recovery if {@link R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour}
+     * is set to 0
+     */
+    @Test
+    public void testTimeWindowLimiting_NativeFailureOff() {
+        when(mClock.getElapsedSinceBootMillis()).thenReturn(0L);
+        mResources.setInteger(R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour, 0);
+        mSelfRecovery.trigger(SelfRecovery.REASON_WIFINATIVE_FAILURE);
+        verify(mActiveModeWarden, never())
+                .recoveryRestartWifi(SelfRecovery.REASON_WIFINATIVE_FAILURE);
+        verify(mActiveModeWarden).recoveryDisableWifi();
+        reset(mActiveModeWarden);
+
+        // Verify L.R.Watchdog can still restart things (It has its own complex limiter)
+        mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
+        verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
+    }
+
+    /**
      * Verifies that invocations of {@link SelfRecovery#trigger(int)} for
      * REASON_LAST_RESORT_WATCHDOG are NOT limited to
-     * {@link SelfRecovery#MAX_RESTARTS_IN_TIME_WINDOW} in a
-     * {@link SelfRecovery#MAX_RESTARTS_TIME_WINDOW_MILLIS} millisecond time window.
+     * {{@link R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour} in a 1 hour time window.
      */
     @Test
     public void testTimeWindowLimiting_lastResortWatchdog_noEffect() {
-        for (int i = 0; i < SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW * 2; i++) {
+        for (int i = 0; i < DEFAULT_MAX_RECOVERY_PER_HOUR * 2; i++) {
             // Verify L.R.Watchdog can still restart things (It has it's own complex limiter)
             mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
             verify(mActiveModeWarden).recoveryRestartWifi(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
@@ -159,12 +183,11 @@ public class SelfRecoveryTest extends WifiBaseTest {
     /**
      * Verifies that invocations of {@link SelfRecovery#trigger(int)} for
      * REASON_STA_IFACE_DOWN are NOT limited to
-     * {@link SelfRecovery#MAX_RESTARTS_IN_TIME_WINDOW} in a
-     * {@link SelfRecovery#MAX_RESTARTS_TIME_WINDOW_MILLIS} millisecond time window.
+     * {{@link R.integer.config_wifiMaxNativeFailureSelfRecoveryPerHour} in a 1 hour time window.
      */
     @Test
     public void testTimeWindowLimiting_staIfaceDown_noEffect() {
-        for (int i = 0; i < SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW * 2; i++) {
+        for (int i = 0; i < DEFAULT_MAX_RECOVERY_PER_HOUR * 2; i++) {
             mSelfRecovery.trigger(SelfRecovery.REASON_STA_IFACE_DOWN);
             verify(mActiveModeWarden).recoveryDisableWifi();
             verify(mActiveModeWarden, never()).recoveryRestartWifi(anyInt());