Limit SelfRecovery wifi restarts to some amount

This prevents wifi from getting into an extreme restart loop if there's
some repeated HAL or WIFICOND crash. Currently limits it to ~48 per day.

Added a buffer logging restarts for HAL_CRASH and WIFICOND_CRASH.
If buffer has more than 2 restarts logged in the last 2 hour time
window, further restarts are ignored. Buffer is cleaned up on every
trigger.

Bug: 62835400
Test: Unit tests
Change-Id: Idb0bc060424423f1a27782868689bff5ed2c0520

1 files changed, 75 insertions, 1 deletions

diff --git a/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java b/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
index 0e55b72a9..c8b93e95d 100644
--- a/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
+++ b/tests/wifitests/src/com/android/server/wifi/SelfRecoveryTest.java
@@ -32,11 +32,12 @@ import org.mockito.Mock;
 public class SelfRecoveryTest {
     SelfRecovery mSelfRecovery;
     @Mock WifiController mWifiController;
+    @Mock Clock mClock;
 
     @Before
     public void setUp() throws Exception {
         initMocks(this);
-        mSelfRecovery = new SelfRecovery(mWifiController);
+        mSelfRecovery = new SelfRecovery(mWifiController, mClock);
     }
 
     /**
@@ -49,10 +50,14 @@ public class SelfRecoveryTest {
         verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
         reset(mWifiController);
 
+        when(mClock.getElapsedSinceBootMillis())
+                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
         mSelfRecovery.trigger(SelfRecovery.REASON_HAL_CRASH);
         verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
         reset(mWifiController);
 
+        when(mClock.getElapsedSinceBootMillis())
+                .thenReturn(2 * (SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1));
         mSelfRecovery.trigger(SelfRecovery.REASON_WIFICOND_CRASH);
         verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
         reset(mWifiController);
@@ -71,4 +76,73 @@ public class SelfRecoveryTest {
         mSelfRecovery.trigger(8);
         verify(mWifiController, never()).sendMessage(anyInt());
     }
+
+    /**
+     * Verifies that invocations of {@link SelfRecovery#trigger(int)} for REASON_HAL_CRASH &
+     * REASON_WIFICOND_CRASH are limited to {@link SelfRecovery#MAX_RESTARTS_IN_TIME_WINDOW} in a
+     * {@link SelfRecovery#MAX_RESTARTS_TIME_WINDOW_MILLIS} millisecond time window.
+     */
+    @Test
+    public void testTimeWindowLimiting_typicalUse() {
+        when(mClock.getElapsedSinceBootMillis()).thenReturn(0L);
+        // Fill up the SelfRecovery's restart time window buffer, ensure all the restart triggers
+        // aren't ignored
+        for (int i = 0; i < SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW / 2; i++) {
+            mSelfRecovery.trigger(SelfRecovery.REASON_HAL_CRASH);
+            verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+            reset(mWifiController);
+
+            mSelfRecovery.trigger(SelfRecovery.REASON_WIFICOND_CRASH);
+            verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+            reset(mWifiController);
+        }
+        if ((SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW % 2) == 1) {
+            mSelfRecovery.trigger(SelfRecovery.REASON_WIFICOND_CRASH);
+            verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+            reset(mWifiController);
+        }
+
+        // Verify that further attempts to trigger restarts for are ignored
+        mSelfRecovery.trigger(SelfRecovery.REASON_HAL_CRASH);
+        verify(mWifiController, never()).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+        reset(mWifiController);
+
+        mSelfRecovery.trigger(SelfRecovery.REASON_WIFICOND_CRASH);
+        verify(mWifiController, never()).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+        reset(mWifiController);
+
+        // Verify L.R.Watchdog can still restart things (It has its own complex limiter)
+        mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
+        verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+        reset(mWifiController);
+
+        // now TRAVEL FORWARDS IN TIME and ensure that more restarts can occur
+        when(mClock.getElapsedSinceBootMillis())
+                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
+        mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
+        verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+        reset(mWifiController);
+
+        when(mClock.getElapsedSinceBootMillis())
+                .thenReturn(SelfRecovery.MAX_RESTARTS_TIME_WINDOW_MILLIS + 1);
+        mSelfRecovery.trigger(SelfRecovery.REASON_HAL_CRASH);
+        verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+        reset(mWifiController);
+    }
+
+    /**
+     * Verifies that invocations of {@link SelfRecovery#trigger(int)} for
+     * REASON_LAST_RESORT_WATCHDOG are NOT limited to
+     * {@link SelfRecovery#MAX_RESTARTS_IN_TIME_WINDOW} in a
+     * {@link SelfRecovery#MAX_RESTARTS_TIME_WINDOW_MILLIS} millisecond time window.
+     */
+    @Test
+    public void testTimeWindowLimiting_lastResortWatchdog_noEffect() {
+        for (int i = 0; i < SelfRecovery.MAX_RESTARTS_IN_TIME_WINDOW * 2; i++) {
+            // Verify L.R.Watchdog can still restart things (It has it's own complex limiter)
+            mSelfRecovery.trigger(SelfRecovery.REASON_LAST_RESORT_WATCHDOG);
+            verify(mWifiController).sendMessage(eq(WifiController.CMD_RESTART_WIFI));
+            reset(mWifiController);
+        }
+    }
 }