Stabilize Android VPN watchdog telemetry

clients/android/app/build.gradle

@@ -30,8 +30,8 @@ android {
         applicationId "su.cin.rapvpn"
         minSdk 26
         targetSdk 35
-        versionCode 198
+        versionCode 199
-        versionName "0.2.198"
+        versionName "0.2.199"
         buildConfigField "String", "DEFAULT_BACKEND_URL", "\"${normalizeGradleString(defaultBackendUrl)}\""
         buildConfigField "String", "DEFAULT_CLUSTER_ID", "\"${normalizeGradleString(defaultClusterId)}\""
         buildConfigField "String", "DEFAULT_ORGANIZATION_ID", "\"${normalizeGradleString(defaultOrganizationId)}\""

clients/android/app/src/main/java/su/cin/rapvpn/RapVpnService.java

@@ -81,6 +81,9 @@ public class RapVpnService extends VpnService {
     private static final int RUNTIME_WATCHDOG_STALE_ROUNDS_BEFORE_RECOVERY = 3;
     private static final int RUNTIME_WATCHDOG_STALE_SYNACKS_BEFORE_RECOVERY = 4;
     private static final int RUNTIME_WATCHDOG_MAX_STALE_ROUNDS_BEFORE_RECOVERY = 6;
+    private static final int RUNTIME_WATCHDOG_OPEN_RELAY_STALE_ROUNDS_BEFORE_RECOVERY = 15;
+    private static final int RUNTIME_WATCHDOG_OPEN_RELAY_STALE_SYNACKS_BEFORE_RECOVERY = 12;
+    private static final int RUNTIME_WATCHDOG_RECENT_DOWNLINK_GRACE_MS = 30000;
     private static final int RUNTIME_WATCHDOG_RECOVERY_COOLDOWN_MS = 20000;
     private static final int RUNTIME_WATCHDOG_HARD_RESTART_COOLDOWN_MS = 60000;
     private static final int DIAGNOSTIC_WATCHDOG_INTERVAL_MS = 5000;
@@ -222,6 +225,7 @@ public class RapVpnService extends VpnService {
     private volatile long lastRuntimeWatchdogHardRestartAt;
     private volatile long lastRuntimeWatchdogDownlinkPackets;
     private volatile long lastRuntimeWatchdogUplinkPackets;
+    private volatile long lastRuntimeWatchdogDownlinkProgressAt;
     private volatile int runtimeWatchdogStaleRounds;
     private volatile long lastDiagnosticEnsureAt;
     private volatile long lastDiagnosticStatusEnsureAt;
@@ -1369,6 +1373,7 @@ public class RapVpnService extends VpnService {
         lastRuntimeWatchdogHardRestartAt = 0;
         lastRuntimeWatchdogDownlinkPackets = 0;
         lastRuntimeWatchdogUplinkPackets = 0;
+        lastRuntimeWatchdogDownlinkProgressAt = runtimeStartedAt;
         runtimeWatchdogStaleRounds = 0;
         downlinkDroppedPackets.set(0);
         downlinkDroppedBytes.set(0);
@@ -1394,7 +1399,9 @@ public class RapVpnService extends VpnService {
         uplinkReadPps = 0f;
         uplinkSentPps = 0f;
         downlinkReceivedPps = 0f;
-        getSharedPreferences(PREFS, MODE_PRIVATE).edit()
+        SharedPreferences.Editor editor = getSharedPreferences(PREFS, MODE_PRIVATE).edit();
+        clearRuntimeDiagnosticPrefs(editor);
+        editor
                 .putString("state", "resetting")
                 .putString("message", "runtime counters reset")
                 .putLong("updated_at", runtimeStartedAt)
@@ -1429,6 +1436,7 @@ public class RapVpnService extends VpnService {
                 .putLong("local_dns_errors", 0)
                 .putLong("runtime_watchdog_recoveries", 0)
                 .putLong("tcp_handshake_stalls", 0)
+                .putLong("runtime_watchdog_hard_restarts", 0)
                 .putInt("uplink_worker_count", 0)
                 .putString("uplink_queue_depths", "")
                 .putInt("uplink_queue_depth_max", 0)
@@ -1443,6 +1451,45 @@ public class RapVpnService extends VpnService {
                 .apply();
     }
 
+    private void clearRuntimeDiagnosticPrefs(SharedPreferences.Editor editor) {
+        String[] prefixes = new String[]{
+                "uplink",
+                "uplink_sender",
+                "uplink_tcp",
+                "downlink",
+                "downlink_tcp",
+                "downlink_writer",
+                "relay",
+                "watchdog"
+        };
+        String[] suffixes = new String[]{
+                "state",
+                "message",
+                "updated_at",
+                "thread_alive",
+                "packets",
+                "errors",
+                "bytes",
+                "rate_mbps",
+                "rate_pps",
+                "error_type"
+        };
+        for (String prefix : prefixes) {
+            for (String suffix : suffixes) {
+                editor.remove(prefix + "_" + suffix);
+            }
+        }
+        for (int i = 0; i < 8; i++) {
+            editor.remove("uplink_queue_" + i + "_offers");
+            editor.remove("uplink_queue_" + i + "_drops");
+            editor.remove("uplink_sender_worker_packets_" + i);
+            editor.remove("uplink_sender_worker_errors_" + i);
+            editor.remove("downlink_queue_" + i + "_offers");
+            editor.remove("downlink_queue_" + i + "_drops");
+            editor.remove("downlink_writer_flow_packets_" + i);
+        }
+    }
+
     private static AtomicLong[] createAtomicCounters(int count) {
         AtomicLong[] values = new AtomicLong[count];
         for (int i = 0; i < count; i++) {
@@ -1555,6 +1602,9 @@ public class RapVpnService extends VpnService {
                 boolean uplinkProgressed = uplinkPackets > lastRuntimeWatchdogUplinkPackets;
                 lastRuntimeWatchdogDownlinkPackets = downlinkPackets;
                 lastRuntimeWatchdogUplinkPackets = uplinkPackets;
+                if (downlinkProgressed) {
+                    lastRuntimeWatchdogDownlinkProgressAt = now;
+                }
                 if (stale <= 0) {
                     runtimeWatchdogStaleRounds = 0;
                     continue;
@@ -1565,6 +1615,17 @@ public class RapVpnService extends VpnService {
                     continue;
                 }
                 runtimeWatchdogStaleRounds++;
+                boolean relayOpen = isPacketWebSocketRelayOpen();
+                boolean recentDownlink = lastRuntimeWatchdogDownlinkProgressAt > 0
+                        && now - lastRuntimeWatchdogDownlinkProgressAt < RUNTIME_WATCHDOG_RECENT_DOWNLINK_GRACE_MS;
+                boolean recentUplinkSendError = lastUplinkSendErrorMessage != null && !lastUplinkSendErrorMessage.isEmpty();
+                if (relayOpen && !recentUplinkSendError
+                        && (recentDownlink
+                        || stale < RUNTIME_WATCHDOG_OPEN_RELAY_STALE_SYNACKS_BEFORE_RECOVERY
+                        || runtimeWatchdogStaleRounds < RUNTIME_WATCHDOG_OPEN_RELAY_STALE_ROUNDS_BEFORE_RECOVERY)) {
+                    writeRuntimeDetail("watchdog_open_relay_waiting", "stale=" + stale + " rounds=" + runtimeWatchdogStaleRounds + " relay_open=true recent_downlink=" + recentDownlink + " uplink_progress=" + uplinkProgressed, "watchdog", runtimeWatchdogRecoveries.get(), tcpHandshakeStalls.get(), "", -1);
+                    continue;
+                }
                 if (runtimeWatchdogStaleRounds < RUNTIME_WATCHDOG_STALE_ROUNDS_BEFORE_RECOVERY
                         || (stale < RUNTIME_WATCHDOG_STALE_SYNACKS_BEFORE_RECOVERY
                         && runtimeWatchdogStaleRounds < RUNTIME_WATCHDOG_MAX_STALE_ROUNDS_BEFORE_RECOVERY)) {
@@ -1593,6 +1654,11 @@ public class RapVpnService extends VpnService {
         }
     }
 
+    private boolean isPacketWebSocketRelayOpen() {
+        VpnPacketWebSocketRelay relay = packetWebSocketRelay;
+        return relay != null && relay.isOpen();
+    }
+
     private void runDiagnosticServiceWatchdog() {
         getSharedPreferences(PREFS, MODE_PRIVATE).edit()
                 .putLong("diagnostic_watchdog_started_at", System.currentTimeMillis())