Foreground/background network stats using sets.

Teach NetworkStats about "counter sets" coming from kernel, and use
them to track usage in foreground/background.  Add AID_NET_BW_ACCT to
system_server so it can control counter sets.

Move to composite key of NetworkIdentitySet, UID, set, and tag when
recording historical usage.  Persisting still clusters by identity,
since that is heaviest object.

Request async stats poll during systemReady() to bootstrap later
delta calculations. Reset kernel counters when UID removed. Update
various tests.

Bug: 5105592, 5146067
Change-Id: Idabec9e3ffcaf212879821515602ecde0a03de8c

6 files changed, 106 insertions, 32 deletions

diff --git a/core/java/android/net/INetworkStatsService.aidl b/core/java/android/net/INetworkStatsService.aidl
index b65506c411e8..0e883cf8ac55 100644
--- a/core/java/android/net/INetworkStatsService.aidl
+++ b/core/java/android/net/INetworkStatsService.aidl
@@ -26,7 +26,7 @@ interface INetworkStatsService {
     /** Return historical network layer stats for traffic that matches template. */
     NetworkStatsHistory getHistoryForNetwork(in NetworkTemplate template, int fields);
     /** Return historical network layer stats for specific UID traffic that matches template. */
-    NetworkStatsHistory getHistoryForUid(in NetworkTemplate template, int uid, int tag, int fields);
+    NetworkStatsHistory getHistoryForUid(in NetworkTemplate template, int uid, int set, int tag, int fields);
 
     /** Return network layer usage summary for traffic that matches template. */
     NetworkStats getSummaryForNetwork(in NetworkTemplate template, long start, long end);
@@ -38,6 +38,8 @@ interface INetworkStatsService {
     /** Increment data layer count of operations performed for UID and tag. */
     void incrementOperationCount(int uid, int tag, int operationCount);
 
+    /** Mark given UID as being in foreground for stats purposes. */
+    void setUidForeground(int uid, boolean uidForeground);
     /** Force update of statistics. */
     void forceUpdate();
 
diff --git a/core/java/android/net/NetworkStats.java b/core/java/android/net/NetworkStats.java
index f2fcb8fad852..272545d0dc5d 100644
--- a/core/java/android/net/NetworkStats.java
+++ b/core/java/android/net/NetworkStats.java
@@ -42,7 +42,13 @@ public class NetworkStats implements Parcelable {
     public static final String IFACE_ALL = null;
     /** {@link #uid} value when UID details unavailable. */
     public static final int UID_ALL = -1;
-    /** {@link #tag} value for without tag. */
+    /** {@link #set} value when all sets combined. */
+    public static final int SET_ALL = -1;
+    /** {@link #set} value where background data is accounted. */
+    public static final int SET_DEFAULT = 0;
+    /** {@link #set} value where foreground data is accounted. */
+    public static final int SET_FOREGROUND = 1;
+    /** {@link #tag} value for total data across all tags. */
     public static final int TAG_NONE = 0;
 
     /**
@@ -53,6 +59,7 @@ public class NetworkStats implements Parcelable {
     private int size;
     private String[] iface;
     private int[] uid;
+    private int[] set;
     private int[] tag;
     private long[] rxBytes;
     private long[] rxPackets;
@@ -63,6 +70,7 @@ public class NetworkStats implements Parcelable {
     public static class Entry {
         public String iface;
         public int uid;
+        public int set;
         public int tag;
         public long rxBytes;
         public long rxPackets;
@@ -71,17 +79,19 @@ public class NetworkStats implements Parcelable {
         public long operations;
 
         public Entry() {
-            this(IFACE_ALL, UID_ALL, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
+            this(IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
         }
 
         public Entry(long rxBytes, long rxPackets, long txBytes, long txPackets, long operations) {
-            this(IFACE_ALL, UID_ALL, TAG_NONE, rxBytes, rxPackets, txBytes, txPackets, operations);
+            this(IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, rxBytes, rxPackets, txBytes, txPackets,
+                    operations);
         }
 
-        public Entry(String iface, int uid, int tag, long rxBytes, long rxPackets, long txBytes,
-                long txPackets, long operations) {
+        public Entry(String iface, int uid, int set, int tag, long rxBytes, long rxPackets,
+                long txBytes, long txPackets, long operations) {
             this.iface = iface;
             this.uid = uid;
+            this.set = set;
             this.tag = tag;
             this.rxBytes = rxBytes;
             this.rxPackets = rxPackets;
@@ -96,6 +106,7 @@ public class NetworkStats implements Parcelable {
         this.size = 0;
         this.iface = new String[initialSize];
         this.uid = new int[initialSize];
+        this.set = new int[initialSize];
         this.tag = new int[initialSize];
         this.rxBytes = new long[initialSize];
         this.rxPackets = new long[initialSize];
@@ -109,6 +120,7 @@ public class NetworkStats implements Parcelable {
         size = parcel.readInt();
         iface = parcel.createStringArray();
         uid = parcel.createIntArray();
+        set = parcel.createIntArray();
         tag = parcel.createIntArray();
         rxBytes = parcel.createLongArray();
         rxPackets = parcel.createLongArray();
@@ -123,6 +135,7 @@ public class NetworkStats implements Parcelable {
         dest.writeInt(size);
         dest.writeStringArray(iface);
         dest.writeIntArray(uid);
+        dest.writeIntArray(set);
         dest.writeIntArray(tag);
         dest.writeLongArray(rxBytes);
         dest.writeLongArray(rxPackets);
@@ -131,15 +144,18 @@ public class NetworkStats implements Parcelable {
         dest.writeLongArray(operations);
     }
 
-    public NetworkStats addValues(String iface, int uid, int tag, long rxBytes, long rxPackets,
-            long txBytes, long txPackets) {
-        return addValues(iface, uid, tag, rxBytes, rxPackets, txBytes, txPackets, 0L);
+    // @VisibleForTesting
+    public NetworkStats addIfaceValues(
+            String iface, long rxBytes, long rxPackets, long txBytes, long txPackets) {
+        return addValues(
+                iface, UID_ALL, SET_DEFAULT, TAG_NONE, rxBytes, rxPackets, txBytes, txPackets, 0L);
     }
 
-    public NetworkStats addValues(String iface, int uid, int tag, long rxBytes, long rxPackets,
-            long txBytes, long txPackets, long operations) {
-        return addValues(
-                new Entry(iface, uid, tag, rxBytes, rxPackets, txBytes, txPackets, operations));
+    // @VisibleForTesting
+    public NetworkStats addValues(String iface, int uid, int set, int tag, long rxBytes,
+            long rxPackets, long txBytes, long txPackets, long operations) {
+        return addValues(new Entry(
+                iface, uid, set, tag, rxBytes, rxPackets, txBytes, txPackets, operations));
     }
 
     /**
@@ -151,6 +167,7 @@ public class NetworkStats implements Parcelable {
             final int newLength = Math.max(iface.length, 10) * 3 / 2;
             iface = Arrays.copyOf(iface, newLength);
             uid = Arrays.copyOf(uid, newLength);
+            set = Arrays.copyOf(set, newLength);
             tag = Arrays.copyOf(tag, newLength);
             rxBytes = Arrays.copyOf(rxBytes, newLength);
             rxPackets = Arrays.copyOf(rxPackets, newLength);
@@ -161,6 +178,7 @@ public class NetworkStats implements Parcelable {
 
         iface[size] = entry.iface;
         uid[size] = entry.uid;
+        set[size] = entry.set;
         tag[size] = entry.tag;
         rxBytes[size] = entry.rxBytes;
         rxPackets[size] = entry.rxPackets;
@@ -179,6 +197,7 @@ public class NetworkStats implements Parcelable {
         final Entry entry = recycle != null ? recycle : new Entry();
         entry.iface = iface[i];
         entry.uid = uid[i];
+        entry.set = set[i];
         entry.tag = tag[i];
         entry.rxBytes = rxBytes[i];
         entry.rxPackets = rxPackets[i];
@@ -201,19 +220,26 @@ public class NetworkStats implements Parcelable {
         return iface.length;
     }
 
+    @Deprecated
     public NetworkStats combineValues(String iface, int uid, int tag, long rxBytes, long rxPackets,
             long txBytes, long txPackets, long operations) {
         return combineValues(
-                new Entry(iface, uid, tag, rxBytes, rxPackets, txBytes, txPackets, operations));
+                iface, uid, SET_DEFAULT, tag, rxBytes, rxPackets, txBytes, txPackets, operations);
+    }
+
+    public NetworkStats combineValues(String iface, int uid, int set, int tag, long rxBytes,
+            long rxPackets, long txBytes, long txPackets, long operations) {
+        return combineValues(new Entry(
+                iface, uid, set, tag, rxBytes, rxPackets, txBytes, txPackets, operations));
     }
 
     /**
      * Combine given values with an existing row, or create a new row if
-     * {@link #findIndex(String, int, int)} is unable to find match. Can also be
-     * used to subtract values from existing rows.
+     * {@link #findIndex(String, int, int, int)} is unable to find match. Can
+     * also be used to subtract values from existing rows.
      */
     public NetworkStats combineValues(Entry entry) {
-        final int i = findIndex(entry.iface, entry.uid, entry.tag);
+        final int i = findIndex(entry.iface, entry.uid, entry.set, entry.tag);
         if (i == -1) {
             // only create new entry when positive contribution
             addValues(entry);
@@ -230,9 +256,10 @@ public class NetworkStats implements Parcelable {
     /**
      * Find first stats index that matches the requested parameters.
      */
-    public int findIndex(String iface, int uid, int tag) {
+    public int findIndex(String iface, int uid, int set, int tag) {
         for (int i = 0; i < size; i++) {
-            if (Objects.equal(iface, this.iface[i]) && uid == this.uid[i] && tag == this.tag[i]) {
+            if (Objects.equal(iface, this.iface[i]) && uid == this.uid[i] && set == this.set[i]
+                    && tag == this.tag[i]) {
                 return i;
             }
         }
@@ -246,7 +273,7 @@ public class NetworkStats implements Parcelable {
      */
     public void spliceOperationsFrom(NetworkStats stats) {
         for (int i = 0; i < size; i++) {
-            final int j = stats.findIndex(IFACE_ALL, uid[i], tag[i]);
+            final int j = stats.findIndex(IFACE_ALL, uid[i], set[i], tag[i]);
             if (j == -1) {
                 operations[i] = 0;
             } else {
@@ -332,10 +359,11 @@ public class NetworkStats implements Parcelable {
         for (int i = 0; i < size; i++) {
             entry.iface = iface[i];
             entry.uid = uid[i];
+            entry.set = set[i];
             entry.tag = tag[i];
 
             // find remote row that matches, and subtract
-            final int j = value.findIndex(entry.iface, entry.uid, entry.tag);
+            final int j = value.findIndex(entry.iface, entry.uid, entry.set, entry.tag);
             if (j == -1) {
                 // newly appearing row, return entire value
                 entry.rxBytes = rxBytes[i];
@@ -377,7 +405,8 @@ public class NetworkStats implements Parcelable {
             pw.print(prefix);
             pw.print("  iface="); pw.print(iface[i]);
             pw.print(" uid="); pw.print(uid[i]);
-            pw.print(" tag=0x"); pw.print(Integer.toHexString(tag[i]));
+            pw.print(" set="); pw.print(setToString(set[i]));
+            pw.print(" tag="); pw.print(tagToString(tag[i]));
             pw.print(" rxBytes="); pw.print(rxBytes[i]);
             pw.print(" rxPackets="); pw.print(rxPackets[i]);
             pw.print(" txBytes="); pw.print(txBytes[i]);
@@ -386,6 +415,29 @@ public class NetworkStats implements Parcelable {
         }
     }
 
+    /**
+     * Return text description of {@link #set} value.
+     */
+    public static String setToString(int set) {
+        switch (set) {
+            case SET_ALL:
+                return "ALL";
+            case SET_DEFAULT:
+                return "DEFAULT";
+            case SET_FOREGROUND:
+                return "FOREGROUND";
+            default:
+                return "UNKNOWN";
+        }
+    }
+
+    /**
+     * Return text description of {@link #tag} value.
+     */
+    public static String tagToString(int tag) {
+        return "0x" + Integer.toHexString(tag);
+    }
+
     @Override
     public String toString() {
         final CharArrayWriter writer = new CharArrayWriter();
diff --git a/core/java/android/net/NetworkStatsHistory.java b/core/java/android/net/NetworkStatsHistory.java
index 4ba44cac296f..b4f15acff1c9 100644
--- a/core/java/android/net/NetworkStatsHistory.java
+++ b/core/java/android/net/NetworkStatsHistory.java
@@ -17,6 +17,7 @@
 package android.net;
 
 import static android.net.NetworkStats.IFACE_ALL;
+import static android.net.NetworkStats.SET_DEFAULT;
 import static android.net.NetworkStats.TAG_NONE;
 import static android.net.NetworkStats.UID_ALL;
 import static android.net.NetworkStatsHistory.DataStreamUtils.readFullLongArray;
@@ -215,8 +216,8 @@ public class NetworkStatsHistory implements Parcelable {
      */
     @Deprecated
     public void recordData(long start, long end, long rxBytes, long txBytes) {
-        recordData(start, end,
-                new NetworkStats.Entry(IFACE_ALL, UID_ALL, TAG_NONE, rxBytes, 0L, txBytes, 0L, 0L));
+        recordData(start, end, new NetworkStats.Entry(
+                IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, rxBytes, 0L, txBytes, 0L, 0L));
     }
 
     /**
@@ -269,7 +270,7 @@ public class NetworkStatsHistory implements Parcelable {
      */
     public void recordEntireHistory(NetworkStatsHistory input) {
         final NetworkStats.Entry entry = new NetworkStats.Entry(
-                IFACE_ALL, UID_ALL, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
+                IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
         for (int i = 0; i < input.bucketCount; i++) {
             final long start = input.bucketStart[i];
             final long end = start + input.bucketDuration;
@@ -422,7 +423,7 @@ public class NetworkStatsHistory implements Parcelable {
         ensureBuckets(start, end);
 
         final NetworkStats.Entry entry = new NetworkStats.Entry(
-                IFACE_ALL, UID_ALL, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
+                IFACE_ALL, UID_ALL, SET_DEFAULT, TAG_NONE, 0L, 0L, 0L, 0L, 0L);
         final Random r = new Random();
         while (rxBytes > 1024 || rxPackets > 128 || txBytes > 1024 || txPackets > 128
                 || operations > 32) {
diff --git a/core/java/android/net/TrafficStats.java b/core/java/android/net/TrafficStats.java
index f138e49140fa..c2c5c183d337 100644
--- a/core/java/android/net/TrafficStats.java
+++ b/core/java/android/net/TrafficStats.java
@@ -205,10 +205,6 @@ public class TrafficStats {
      * @param operationCount Number of operations to increment count by.
      */
     public static void incrementOperationCount(int tag, int operationCount) {
-        if (operationCount < 0) {
-            throw new IllegalArgumentException("operation count can only be incremented");
-        }
-
         final INetworkStatsService statsService = INetworkStatsService.Stub.asInterface(
                 ServiceManager.getService(Context.NETWORK_STATS_SERVICE));
         final int uid = android.os.Process.myUid();
diff --git a/core/java/com/android/internal/os/ZygoteInit.java b/core/java/com/android/internal/os/ZygoteInit.java
index 16336e0e81b3..9c45dc64f469 100644
--- a/core/java/com/android/internal/os/ZygoteInit.java
+++ b/core/java/com/android/internal/os/ZygoteInit.java
@@ -474,7 +474,7 @@ public class ZygoteInit {
         String args[] = {
             "--setuid=1000",
             "--setgid=1000",
-            "--setgroups=1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1018,3001,3002,3003,3006",
+            "--setgroups=1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1018,3001,3002,3003,3006,3007",
             "--capabilities=130104352,130104352",
             "--runtime-init",
             "--nice-name=system_server",
diff --git a/core/java/com/android/server/NetworkManagementSocketTagger.java b/core/java/com/android/server/NetworkManagementSocketTagger.java
index 4667e5fc978d..23af37e6188d 100644
--- a/core/java/com/android/server/NetworkManagementSocketTagger.java
+++ b/core/java/com/android/server/NetworkManagementSocketTagger.java
@@ -16,8 +16,11 @@
 
 package com.android.server;
 
+import android.net.NetworkStats;
 import android.os.SystemProperties;
 import android.util.Log;
+import android.util.Slog;
+
 import dalvik.system.SocketTagger;
 import libcore.io.IoUtils;
 
@@ -122,6 +125,26 @@ public final class NetworkManagementSocketTagger extends SocketTagger {
         public int statsUid = -1;
     }
 
+    public static void setKernelCounterSet(int uid, int counterSet) {
+        final StringBuilder command = new StringBuilder();
+        command.append("s ").append(counterSet).append(" ").append(uid);
+        try {
+            internalModuleCtrl(command.toString());
+        } catch (IOException e) {
+            Slog.w(TAG, "problem changing counter set for uid " + uid + " : " + e);
+        }
+    }
+
+    public static void resetKernelUidStats(int uid) {
+        final StringBuilder command = new StringBuilder();
+        command.append("d 0 ").append(uid);
+        try {
+            internalModuleCtrl(command.toString());
+        } catch (IOException e) {
+            Slog.w(TAG, "problem clearing counters for uid " + uid + " : " + e);
+        }
+    }
+
     /**
      * Sends commands to the kernel netfilter module.
      *
@@ -141,7 +164,7 @@ public final class NetworkManagementSocketTagger extends SocketTagger {
      *   <li><i>*_tag</i> are 64bit values</li></ul>
      *
      */
-    private void internalModuleCtrl(String cmd) throws IOException {
+    private static void internalModuleCtrl(String cmd) throws IOException {
         if (!SystemProperties.getBoolean(PROP_QTAGUID_ENABLED, false)) return;
 
         // TODO: migrate to native library for tagging commands