Merge "Rename dscp_policy.o to dscpPolicy.o"

Tethering/apex/Android.bp

@@ -85,7 +85,7 @@ apex {
     bpfs: [
         "block.o",
         "clatd.o",
-        "dscp_policy.o",
+        "dscpPolicy.o",
         "netd.o",
         "offload.o",
         "offload@btf.o",
@@ -209,7 +209,7 @@ override_apex {
     bpfs: [
         "block.o",
         "clatd.o",
-        "dscp_policy.o",
+        "dscpPolicy.o",
         "netd.o",
         "offload@inprocess.o",
         "test@inprocess.o",

bpf_progs/Android.bp

@@ -72,8 +72,8 @@ bpf {
 }
 
 bpf {
-    name: "dscp_policy.o",
+    name: "dscpPolicy.o",
-    srcs: ["dscp_policy.c"],
+    srcs: ["dscpPolicy.c"],
     btf: true,
     cflags: [
         "-Wall",

bpf_progs/dscpPolicy.c

@@ -0,0 +1,319 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <linux/bpf.h>
+#include <linux/if_ether.h>
+#include <linux/if_packet.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/pkt_cls.h>
+#include <linux/tcp.h>
+#include <linux/types.h>
+#include <netinet/in.h>
+#include <netinet/udp.h>
+#include <stdint.h>
+#include <string.h>
+
+// The resulting .o needs to load on the Android T beta 3 bpfloader
+#define BPFLOADER_MIN_VER BPFLOADER_T_BETA3_VERSION
+
+#include "bpf_helpers.h"
+#include "dscpPolicy.h"
+
+#define ECN_MASK 3
+#define IP4_OFFSET(field, header) (header + offsetof(struct iphdr, field))
+#define UPDATE_TOS(dscp, tos) (dscp << 2) | (tos & ECN_MASK)
+#define UPDATE_PRIORITY(dscp) ((dscp >> 2) + 0x60)
+#define UPDATE_FLOW_LABEL(dscp, flow_lbl) ((dscp & 0xf) << 6) + (flow_lbl >> 6)
+
+DEFINE_BPF_MAP_GRW(switch_comp_map, ARRAY, int, uint64_t, 1, AID_SYSTEM)
+
+DEFINE_BPF_MAP_GRW(ipv4_socket_to_policies_map_A, HASH, uint64_t, RuleEntry, MAX_POLICIES,
+                   AID_SYSTEM)
+DEFINE_BPF_MAP_GRW(ipv4_socket_to_policies_map_B, HASH, uint64_t, RuleEntry, MAX_POLICIES,
+                   AID_SYSTEM)
+DEFINE_BPF_MAP_GRW(ipv6_socket_to_policies_map_A, HASH, uint64_t, RuleEntry, MAX_POLICIES,
+                   AID_SYSTEM)
+DEFINE_BPF_MAP_GRW(ipv6_socket_to_policies_map_B, HASH, uint64_t, RuleEntry, MAX_POLICIES,
+                   AID_SYSTEM)
+
+DEFINE_BPF_MAP_GRW(ipv4_dscp_policies_map, ARRAY, uint32_t, DscpPolicy, MAX_POLICIES, AID_SYSTEM)
+DEFINE_BPF_MAP_GRW(ipv6_dscp_policies_map, ARRAY, uint32_t, DscpPolicy, MAX_POLICIES, AID_SYSTEM)
+
+static inline __always_inline void match_policy(struct __sk_buff* skb, bool ipv4, bool is_eth) {
+    void* data = (void*)(long)skb->data;
+    const void* data_end = (void*)(long)skb->data_end;
+
+    const int l2_header_size = is_eth ? sizeof(struct ethhdr) : 0;
+    struct ethhdr* eth = is_eth ? data : NULL;
+
+    if (data + l2_header_size > data_end) return;
+
+    int zero = 0;
+    int hdr_size = 0;
+    uint64_t* selected_map = bpf_switch_comp_map_lookup_elem(&zero);
+
+    // use this with HASH map so map lookup only happens once policies have been added?
+    if (!selected_map) {
+        return;
+    }
+
+    // used for map lookup
+    uint64_t cookie = bpf_get_socket_cookie(skb);
+    if (!cookie) return;
+
+    uint16_t sport = 0;
+    uint16_t dport = 0;
+    uint8_t protocol = 0;  // TODO: Use are reserved value? Or int (-1) and cast to uint below?
+    struct in6_addr src_ip = {};
+    struct in6_addr dst_ip = {};
+    uint8_t tos = 0;       // Only used for IPv4
+    uint8_t priority = 0;  // Only used for IPv6
+    uint8_t flow_lbl = 0;  // Only used for IPv6
+    if (ipv4) {
+        const struct iphdr* const iph = is_eth ? (void*)(eth + 1) : data;
+        hdr_size = l2_header_size + sizeof(struct iphdr);
+        // Must have ipv4 header
+        if (data + hdr_size > data_end) return;
+
+        // IP version must be 4
+        if (iph->version != 4) return;
+
+        // We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
+        if (iph->ihl != 5) return;
+
+        // V4 mapped address in in6_addr sets 10/11 position to 0xff.
+        src_ip.s6_addr32[2] = htonl(0x0000ffff);
+        dst_ip.s6_addr32[2] = htonl(0x0000ffff);
+
+        // Copy IPv4 address into in6_addr for easy comparison below.
+        src_ip.s6_addr32[3] = iph->saddr;
+        dst_ip.s6_addr32[3] = iph->daddr;
+        protocol = iph->protocol;
+        tos = iph->tos;
+    } else {
+        struct ipv6hdr* ip6h = is_eth ? (void*)(eth + 1) : data;
+        hdr_size = l2_header_size + sizeof(struct ipv6hdr);
+        // Must have ipv6 header
+        if (data + hdr_size > data_end) return;
+
+        if (ip6h->version != 6) return;
+
+        src_ip = ip6h->saddr;
+        dst_ip = ip6h->daddr;
+        protocol = ip6h->nexthdr;
+        priority = ip6h->priority;
+        flow_lbl = ip6h->flow_lbl[0];
+    }
+
+    switch (protocol) {
+        case IPPROTO_UDP:
+        case IPPROTO_UDPLITE: {
+            struct udphdr* udp;
+            udp = data + hdr_size;
+            if ((void*)(udp + 1) > data_end) return;
+            sport = udp->source;
+            dport = udp->dest;
+        } break;
+        case IPPROTO_TCP: {
+            struct tcphdr* tcp;
+            tcp = data + hdr_size;
+            if ((void*)(tcp + 1) > data_end) return;
+            sport = tcp->source;
+            dport = tcp->dest;
+        } break;
+        default:
+            return;
+    }
+
+    RuleEntry* existing_rule;
+    if (ipv4) {
+        if (*selected_map == MAP_A) {
+            existing_rule = bpf_ipv4_socket_to_policies_map_A_lookup_elem(&cookie);
+        } else {
+            existing_rule = bpf_ipv4_socket_to_policies_map_B_lookup_elem(&cookie);
+        }
+    } else {
+        if (*selected_map == MAP_A) {
+            existing_rule = bpf_ipv6_socket_to_policies_map_A_lookup_elem(&cookie);
+        } else {
+            existing_rule = bpf_ipv6_socket_to_policies_map_B_lookup_elem(&cookie);
+        }
+    }
+
+    if (existing_rule && v6_equal(src_ip, existing_rule->src_ip) &&
+            v6_equal(dst_ip, existing_rule->dst_ip) && skb->ifindex == existing_rule->ifindex &&
+        ntohs(sport) == htons(existing_rule->src_port) &&
+        ntohs(dport) == htons(existing_rule->dst_port) && protocol == existing_rule->proto) {
+        if (ipv4) {
+            uint8_t newTos = UPDATE_TOS(existing_rule->dscp_val, tos);
+            bpf_l3_csum_replace(skb, IP4_OFFSET(check, l2_header_size), htons(tos), htons(newTos),
+                                sizeof(uint16_t));
+            bpf_skb_store_bytes(skb, IP4_OFFSET(tos, l2_header_size), &newTos, sizeof(newTos), 0);
+        } else {
+            uint8_t new_priority = UPDATE_PRIORITY(existing_rule->dscp_val);
+            uint8_t new_flow_label = UPDATE_FLOW_LABEL(existing_rule->dscp_val, flow_lbl);
+            bpf_skb_store_bytes(skb, 0 + l2_header_size, &new_priority, sizeof(uint8_t), 0);
+            bpf_skb_store_bytes(skb, 1 + l2_header_size, &new_flow_label, sizeof(uint8_t), 0);
+        }
+        return;
+    }
+
+    // Linear scan ipv4_dscp_policies_map since no stored params match skb.
+    int best_score = -1;
+    uint32_t best_match = 0;
+
+    for (register uint64_t i = 0; i < MAX_POLICIES; i++) {
+        int score = 0;
+        uint8_t temp_mask = 0;
+        // Using a uint64 in for loop prevents infinite loop during BPF load,
+        // but the key is uint32, so convert back.
+        uint32_t key = i;
+
+        DscpPolicy* policy;
+        if (ipv4) {
+            policy = bpf_ipv4_dscp_policies_map_lookup_elem(&key);
+        } else {
+            policy = bpf_ipv6_dscp_policies_map_lookup_elem(&key);
+        }
+
+        // If the policy lookup failed, present_fields is 0, or iface index does not match
+        // index on skb buff, then we can continue to next policy.
+        if (!policy || policy->present_fields == 0 || policy->ifindex != skb->ifindex) continue;
+
+        if ((policy->present_fields & SRC_IP_MASK_FLAG) == SRC_IP_MASK_FLAG &&
+            v6_equal(src_ip, policy->src_ip)) {
+            score++;
+            temp_mask |= SRC_IP_MASK_FLAG;
+        }
+        if ((policy->present_fields & DST_IP_MASK_FLAG) == DST_IP_MASK_FLAG &&
+            v6_equal(dst_ip, policy->dst_ip)) {
+            score++;
+            temp_mask |= DST_IP_MASK_FLAG;
+        }
+        if ((policy->present_fields & SRC_PORT_MASK_FLAG) == SRC_PORT_MASK_FLAG &&
+            ntohs(sport) == htons(policy->src_port)) {
+            score++;
+            temp_mask |= SRC_PORT_MASK_FLAG;
+        }
+        if ((policy->present_fields & DST_PORT_MASK_FLAG) == DST_PORT_MASK_FLAG &&
+            ntohs(dport) >= htons(policy->dst_port_start) &&
+            ntohs(dport) <= htons(policy->dst_port_end)) {
+            score++;
+            temp_mask |= DST_PORT_MASK_FLAG;
+        }
+        if ((policy->present_fields & PROTO_MASK_FLAG) == PROTO_MASK_FLAG &&
+            protocol == policy->proto) {
+            score++;
+            temp_mask |= PROTO_MASK_FLAG;
+        }
+
+        if (score > best_score && temp_mask == policy->present_fields) {
+            best_match = i;
+            best_score = score;
+        }
+    }
+
+    uint8_t new_tos = 0;  // Can 0 be used as default forwarding value?
+    uint8_t new_dscp = 0;
+    uint8_t new_priority = 0;
+    uint8_t new_flow_lbl = 0;
+    if (best_score > 0) {
+        DscpPolicy* policy;
+        if (ipv4) {
+            policy = bpf_ipv4_dscp_policies_map_lookup_elem(&best_match);
+        } else {
+            policy = bpf_ipv6_dscp_policies_map_lookup_elem(&best_match);
+        }
+
+        if (policy) {
+            new_dscp = policy->dscp_val;
+            if (ipv4) {
+                new_tos = UPDATE_TOS(new_dscp, tos);
+            } else {
+                new_priority = UPDATE_PRIORITY(new_dscp);
+                new_flow_lbl = UPDATE_FLOW_LABEL(new_dscp, flow_lbl);
+            }
+        }
+    } else
+        return;
+
+    RuleEntry value = {
+        .src_ip = src_ip,
+        .dst_ip = dst_ip,
+        .ifindex = skb->ifindex,
+        .src_port = sport,
+        .dst_port = dport,
+        .proto = protocol,
+        .dscp_val = new_dscp,
+    };
+
+    // Update map with new policy.
+    if (ipv4) {
+        if (*selected_map == MAP_A) {
+            bpf_ipv4_socket_to_policies_map_A_update_elem(&cookie, &value, BPF_ANY);
+        } else {
+            bpf_ipv4_socket_to_policies_map_B_update_elem(&cookie, &value, BPF_ANY);
+        }
+    } else {
+        if (*selected_map == MAP_A) {
+            bpf_ipv6_socket_to_policies_map_A_update_elem(&cookie, &value, BPF_ANY);
+        } else {
+            bpf_ipv6_socket_to_policies_map_B_update_elem(&cookie, &value, BPF_ANY);
+        }
+    }
+
+    // Need to store bytes after updating map or program will not load.
+    if (ipv4 && new_tos != (tos & 252)) {
+        bpf_l3_csum_replace(skb, IP4_OFFSET(check, l2_header_size), htons(tos), htons(new_tos), 2);
+        bpf_skb_store_bytes(skb, IP4_OFFSET(tos, l2_header_size), &new_tos, sizeof(new_tos), 0);
+    } else if (!ipv4 && (new_priority != priority || new_flow_lbl != flow_lbl)) {
+        bpf_skb_store_bytes(skb, l2_header_size, &new_priority, sizeof(new_priority), 0);
+        bpf_skb_store_bytes(skb, l2_header_size + 1, &new_flow_lbl, sizeof(new_flow_lbl), 0);
+    }
+    return;
+}
+
+DEFINE_BPF_PROG_KVER("schedcls/set_dscp_ether", AID_ROOT, AID_SYSTEM,
+                     schedcls_set_dscp_ether, KVER(5, 15, 0))
+(struct __sk_buff* skb) {
+    if (skb->pkt_type != PACKET_HOST) return TC_ACT_PIPE;
+
+    if (skb->protocol == htons(ETH_P_IP)) {
+        match_policy(skb, true, true);
+    } else if (skb->protocol == htons(ETH_P_IPV6)) {
+        match_policy(skb, false, true);
+    }
+
+    // Always return TC_ACT_PIPE
+    return TC_ACT_PIPE;
+}
+
+DEFINE_BPF_PROG_KVER("schedcls/set_dscp_raw_ip", AID_ROOT, AID_SYSTEM,
+                     schedcls_set_dscp_raw_ip, KVER(5, 15, 0))
+(struct __sk_buff* skb) {
+    if (skb->protocol == htons(ETH_P_IP)) {
+        match_policy(skb, true, false);
+    } else if (skb->protocol == htons(ETH_P_IPV6)) {
+        match_policy(skb, false, false);
+    }
+
+    // Always return TC_ACT_PIPE
+    return TC_ACT_PIPE;
+}
+
+LICENSE("Apache 2.0");
+CRITICAL("Connectivity");

bpf_progs/dscpPolicy.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define MAX_POLICIES 16
+#define MAP_A 1
+#define MAP_B 2
+
+#define SRC_IP_MASK_FLAG     1
+#define DST_IP_MASK_FLAG     2
+#define SRC_PORT_MASK_FLAG   4
+#define DST_PORT_MASK_FLAG   8
+#define PROTO_MASK_FLAG      16
+
+#define STRUCT_SIZE(name, size) _Static_assert(sizeof(name) == (size), "Incorrect struct size.")
+
+#define v6_equal(a, b) \
+    (((a.s6_addr32[0] ^ b.s6_addr32[0]) | \
+      (a.s6_addr32[1] ^ b.s6_addr32[1]) | \
+      (a.s6_addr32[2] ^ b.s6_addr32[2]) | \
+      (a.s6_addr32[3] ^ b.s6_addr32[3])) == 0)
+
+// TODO: these are already defined in packages/modules/Connectivity/bpf_progs/bpf_net_helpers.h.
+// smove to common location in future.
+static uint64_t (*bpf_get_socket_cookie)(struct __sk_buff* skb) =
+        (void*)BPF_FUNC_get_socket_cookie;
+static int (*bpf_skb_store_bytes)(struct __sk_buff* skb, __u32 offset, const void* from, __u32 len,
+                                  __u64 flags) = (void*)BPF_FUNC_skb_store_bytes;
+static int (*bpf_l3_csum_replace)(struct __sk_buff* skb, __u32 offset, __u64 from, __u64 to,
+                                  __u64 flags) = (void*)BPF_FUNC_l3_csum_replace;
+static long (*bpf_skb_ecn_set_ce)(struct __sk_buff* skb) =
+        (void*)BPF_FUNC_skb_ecn_set_ce;
+
+typedef struct {
+    struct in6_addr src_ip;
+    struct in6_addr dst_ip;
+    uint32_t ifindex;
+    __be16 src_port;
+    __be16 dst_port_start;
+    __be16 dst_port_end;
+    uint8_t proto;
+    uint8_t dscp_val;
+    uint8_t present_fields;
+    uint8_t pad[3];
+} DscpPolicy;
+STRUCT_SIZE(DscpPolicy, 2 * 16 + 4 + 3 * 2 + 3 * 1 + 3);  // 48
+
+typedef struct {
+    struct in6_addr src_ip;
+    struct in6_addr dst_ip;
+    __u32 ifindex;
+    __be16 src_port;
+    __be16 dst_port;
+    __u8 proto;
+    __u8 dscp_val;
+    __u8 pad[2];
+} RuleEntry;
+STRUCT_SIZE(RuleEntry, 2 * 16 + 1 * 4 + 2 * 2 + 2 * 1 + 2);  // 44

service/src/com/android/server/connectivity/DscpPolicyTracker.java

@@ -52,12 +52,12 @@ public class DscpPolicyTracker {
 
     private static final String TAG = DscpPolicyTracker.class.getSimpleName();
     private static final String PROG_PATH =
-            "/sys/fs/bpf/net_shared/prog_dscp_policy_schedcls_set_dscp";
+            "/sys/fs/bpf/net_shared/prog_dscpPolicy_schedcls_set_dscp";
     // Name is "map + *.o + map_name + map". Can probably shorten this
     private static final String IPV4_POLICY_MAP_PATH = makeMapPath(
-            "dscp_policy_ipv4_dscp_policies");
+            "dscpPolicy_ipv4_dscp_policies");
     private static final String IPV6_POLICY_MAP_PATH = makeMapPath(
-            "dscp_policy_ipv6_dscp_policies");
+            "dscpPolicy_ipv6_dscp_policies");
     private static final int MAX_POLICIES = 16;
 
     private static String makeMapPath(String which) {
@@ -213,7 +213,7 @@ public class DscpPolicyTracker {
     }
 
     /**
-     * Add the provided DSCP policy to the bpf map. Attach bpf program dscp_policy to iface
+     * Add the provided DSCP policy to the bpf map. Attach bpf program dscpPolicy to iface
      * if not already attached. Response will be sent back to nai with status.
      *
      * DSCP_POLICY_STATUS_SUCCESS - if policy was added successfully

tests/mts/bpf_existence_test.cpp

@@ -87,13 +87,13 @@ static const set<string> MAINLINE_FOR_T_PLUS = {
     SHARED "map_block_blocked_ports_map",
     SHARED "map_clatd_clat_egress4_map",
     SHARED "map_clatd_clat_ingress6_map",
-    SHARED "map_dscp_policy_ipv4_dscp_policies_map",
+    SHARED "map_dscpPolicy_ipv4_dscp_policies_map",
-    SHARED "map_dscp_policy_ipv4_socket_to_policies_map_A",
+    SHARED "map_dscpPolicy_ipv4_socket_to_policies_map_A",
-    SHARED "map_dscp_policy_ipv4_socket_to_policies_map_B",
+    SHARED "map_dscpPolicy_ipv4_socket_to_policies_map_B",
-    SHARED "map_dscp_policy_ipv6_dscp_policies_map",
+    SHARED "map_dscpPolicy_ipv6_dscp_policies_map",
-    SHARED "map_dscp_policy_ipv6_socket_to_policies_map_A",
+    SHARED "map_dscpPolicy_ipv6_socket_to_policies_map_A",
-    SHARED "map_dscp_policy_ipv6_socket_to_policies_map_B",
+    SHARED "map_dscpPolicy_ipv6_socket_to_policies_map_B",
-    SHARED "map_dscp_policy_switch_comp_map",
+    SHARED "map_dscpPolicy_switch_comp_map",
     NETD "map_netd_app_uid_stats_map",
     NETD "map_netd_configuration_map",
     NETD "map_netd_cookie_tag_map",
@@ -126,8 +126,8 @@ static const set<string> MAINLINE_FOR_T_5_4_PLUS = {
 
 // Provided by *current* mainline module for T+ devices with 5.15+ kernels
 static const set<string> MAINLINE_FOR_T_5_15_PLUS = {
-    SHARED "prog_dscp_policy_schedcls_set_dscp_ether",
+    SHARED "prog_dscpPolicy_schedcls_set_dscp_ether",
-    SHARED "prog_dscp_policy_schedcls_set_dscp_raw_ip",
+    SHARED "prog_dscpPolicy_schedcls_set_dscp_raw_ip",
 };
 
 void addAll(set<string>* a, const set<string>& b) {