From e8cf3b95869b71ff4719b037f79d74b74d2a2fc3 Mon Sep 17 00:00:00 2001
From: Glen Kuhne <kuh@google.com>
Date: Fri, 23 Jun 2017 14:16:06 -0700
Subject: Limit SelfRecovery wifi restarts to some amount

This prevents wifi from getting into an extreme restart loop if there's
some repeated HAL or WIFICOND crash. Currently limits it to ~48 per day.

Added a buffer logging restarts for HAL_CRASH and WIFICOND_CRASH.
If buffer has more than 2 restarts logged in the last 2 hour time
window, further restarts are ignored. Buffer is cleaned up on every
trigger.

Bug: 62835400
Test: Unit tests
Change-Id: Idb0bc060424423f1a27782868689bff5ed2c0520
---
 .../java/com/android/server/wifi/SelfRecovery.java | 43 ++++++++++++++++++++--
 .../java/com/android/server/wifi/WifiInjector.java |  2 +-
 2 files changed, 40 insertions(+), 5 deletions(-)

(limited to 'service')

diff --git a/service/java/com/android/server/wifi/SelfRecovery.java b/service/java/com/android/server/wifi/SelfRecovery.java
index b35b7ccce..21a3e0ac7 100644
--- a/service/java/com/android/server/wifi/SelfRecovery.java
+++ b/service/java/com/android/server/wifi/SelfRecovery.java
@@ -18,6 +18,9 @@ package com.android.server.wifi;
 
 import android.util.Log;
 
+import java.util.Iterator;
+import java.util.LinkedList;
+
 /**
  * This class is used to recover the wifi stack from a fatal failure. The recovery mechanism
  * involves triggering a stack restart (essentially simulating an airplane mode toggle) using
@@ -36,7 +39,8 @@ public class SelfRecovery {
     public static final int REASON_LAST_RESORT_WATCHDOG = 0;
     public static final int REASON_HAL_CRASH = 1;
     public static final int REASON_WIFICOND_CRASH = 2;
-
+    public static final long MAX_RESTARTS_IN_TIME_WINDOW = 2; // 2 restarts per hour
+    public static final long MAX_RESTARTS_TIME_WINDOW_MILLIS = 60 * 60 * 1000; // 1 hour
     private static final String[] REASON_STRINGS = {
             "Last Resort Watchdog", // REASON_LAST_RESORT_WATCHDOG
             "Hal Crash",            // REASON_HAL_CRASH
@@ -44,9 +48,13 @@ public class SelfRecovery {
     };
 
     private final WifiController mWifiController;
-
-    SelfRecovery(WifiController wifiController) {
+    private final Clock mClock;
+    // Time since boot (in millis) that restart occurred
+    private final LinkedList<Long> mPastRestartTimes;
+    public SelfRecovery(WifiController wifiController, Clock clock) {
         mWifiController = wifiController;
+        mClock = clock;
+        mPastRestartTimes = new LinkedList<Long>();
     }
 
     /**
@@ -59,11 +67,38 @@ public class SelfRecovery {
      * @param reason One of the above |REASON_*| codes.
      */
     public void trigger(int reason) {
-        if (reason < REASON_LAST_RESORT_WATCHDOG || reason > REASON_WIFICOND_CRASH) {
+        if (!(reason == REASON_LAST_RESORT_WATCHDOG || reason == REASON_HAL_CRASH
+                || reason == REASON_WIFICOND_CRASH)) {
             Log.e(TAG, "Invalid trigger reason. Ignoring...");
             return;
         }
         Log.wtf(TAG, "Triggering recovery for reason: " + REASON_STRINGS[reason]);
+        if (reason == REASON_WIFICOND_CRASH || reason == REASON_HAL_CRASH) {
+            trimPastRestartTimes();
+            // Ensure there haven't been too many restarts within MAX_RESTARTS_TIME_WINDOW
+            if (mPastRestartTimes.size() >= MAX_RESTARTS_IN_TIME_WINDOW) {
+                Log.e(TAG, "Already restarted wifi (" + MAX_RESTARTS_IN_TIME_WINDOW + ") times in"
+                        + " last (" + MAX_RESTARTS_TIME_WINDOW_MILLIS + "ms ). Ignoring...");
+                return;
+            }
+            mPastRestartTimes.add(mClock.getElapsedSinceBootMillis());
+        }
         mWifiController.sendMessage(WifiController.CMD_RESTART_WIFI);
     }
+
+    /**
+     * Process the mPastRestartTimes list, removing elements outside the max restarts time window
+     */
+    private void trimPastRestartTimes() {
+        Iterator<Long> iter = mPastRestartTimes.iterator();
+        long now = mClock.getElapsedSinceBootMillis();
+        while (iter.hasNext()) {
+            Long restartTimeMillis = iter.next();
+            if (now - restartTimeMillis > MAX_RESTARTS_TIME_WINDOW_MILLIS) {
+                iter.remove();
+            } else {
+                break;
+            }
+        }
+    }
 }
diff --git a/service/java/com/android/server/wifi/WifiInjector.java b/service/java/com/android/server/wifi/WifiInjector.java
index fd7222e0c..c4be62570 100644
--- a/service/java/com/android/server/wifi/WifiInjector.java
+++ b/service/java/com/android/server/wifi/WifiInjector.java
@@ -230,7 +230,7 @@ public class WifiInjector {
         mLockManager = new WifiLockManager(mContext, BatteryStatsService.getService());
         mWifiController = new WifiController(mContext, mWifiStateMachine, mSettingsStore,
                 mLockManager, mWifiServiceHandlerThread.getLooper(), mFrameworkFacade);
-        mSelfRecovery = new SelfRecovery(mWifiController);
+        mSelfRecovery = new SelfRecovery(mWifiController, mClock);
         mWifiLastResortWatchdog = new WifiLastResortWatchdog(mSelfRecovery, mWifiMetrics);
         mWifiMulticastLockManager = new WifiMulticastLockManager(mWifiStateMachine,
                 BatteryStatsService.getService());
-- 
cgit v1.2.3