joy/android_packages_modules_Connectivity
Watch
1
Fork
You've already forked android_packages_modules_Connectivity
 0
Code Issues Pull requests Projects Releases Wiki Activity

bpf: git mv bpf_progs bpf/progs

Browse source 
Step 1 of moving bpf related stuff into a common directory,
will ease development.

Test: TreeHugger
Signed-off-by: Maciej Żenczykowski <maze@google.com>
Change-Id: Id7a604ec462852b74f2a33a57e99034e0db0c06c
This commit is contained in:
Maciej Żenczykowski 2024-08-19 15:18:40 -07:00
commit 40aecf255d
15 changed files with 0 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

118
bpf/progs/Android.bp Normal file
View file

@ -0,0 +1,118 @@
//
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
//
// struct definitions shared with JNI
//
package {
default_team: "trendy_team_fwk_core_networking",
default_applicable_licenses: ["Android-Apache-2.0"],
}
cc_library_headers {
name: "bpf_connectivity_headers",
vendor_available: false,
host_supported: false,
header_libs: [
"bpf_headers",
"netd_mainline_headers",
],
export_header_lib_headers: [
"bpf_headers",
"netd_mainline_headers",
],
export_include_dirs: ["."],
cflags: [
"-Wall",
"-Werror",
"-Wextra",
],
sdk_version: "30",
min_sdk_version: "30",
apex_available: [
"//apex_available:platform",
"com.android.tethering",
],
visibility: [
"//packages/modules/Connectivity/DnsResolver",
"//packages/modules/Connectivity/netd",
"//packages/modules/Connectivity/service",
"//packages/modules/Connectivity/service/native/libs/libclat",
"//packages/modules/Connectivity/Tethering",
"//packages/modules/Connectivity/service/native",
"//packages/modules/Connectivity/tests/native/connectivity_native_test",
"//packages/modules/Connectivity/tests/native/utilities",
"//packages/modules/Connectivity/service-t/native/libs/libnetworkstats",
"//packages/modules/Connectivity/tests/unit/jni",
],
}
//
// bpf kernel programs
//
bpf {
name: "block.o",
srcs: ["block.c"],
sub_dir: "net_shared",
}
bpf {
name: "dscpPolicy.o",
srcs: ["dscpPolicy.c"],
sub_dir: "net_shared",
}
// Ships to Android S, the bpfloader of which fails to parse BTF enabled .o's.
bpf {
name: "offload.o",
srcs: ["offload.c"],
btf: false,
}
// This version ships to Android T+ which uses mainline netbpfload.
bpf {
name: "offload@mainline.o",
srcs: ["offload@mainline.c"],
cflags: ["-DMAINLINE"],
}
// Ships to Android S, the bpfloader of which fails to parse BTF enabled .o's.
bpf {
name: "test.o",
srcs: ["test.c"],
btf: false,
}
// This version ships to Android T+ which uses mainline netbpfload.
bpf {
name: "test@mainline.o",
srcs: ["test@mainline.c"],
cflags: ["-DMAINLINE"],
}
bpf {
name: "clatd.o",
srcs: ["clatd.c"],
sub_dir: "net_shared",
}
bpf {
// WARNING: Android T's non-updatable netd depends on 'netd' string for xt_bpf programs it loads
name: "netd.o",
srcs: ["netd.c"],
// WARNING: Android T's non-updatable netd depends on 'netd_shared' string for xt_bpf programs
sub_dir: "netd_shared",
}

69
bpf/progs/block.c Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// The resulting .o needs to load on Android T+
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#include "bpf_net_helpers.h"
DEFINE_BPF_MAP_GRW(blocked_ports_map, ARRAY, int, uint64_t,
1024 /* 64K ports -> 1024 u64s */, AID_SYSTEM)
static inline __always_inline int block_port(struct bpf_sock_addr *ctx) {
if (!ctx->user_port) return BPF_ALLOW;
switch (ctx->protocol) {
case IPPROTO_TCP:
case IPPROTO_MPTCP:
case IPPROTO_UDP:
case IPPROTO_UDPLITE:
case IPPROTO_DCCP:
case IPPROTO_SCTP:
break;
default:
return BPF_ALLOW; // unknown protocols are allowed
}
int key = ctx->user_port >> 6;
int shift = ctx->user_port & 63;
uint64_t *val = bpf_blocked_ports_map_lookup_elem(&key);
// Lookup should never fail in reality, but if it does return here to keep the
// BPF verifier happy.
if (!val) return BPF_ALLOW;
if ((*val >> shift) & 1) return BPF_DISALLOW;
return BPF_ALLOW;
}
// the program need to be accessible/loadable by netd (from netd updatable plugin)
#define DEFINE_NETD_RO_BPF_PROG(SECTION_NAME, the_prog, min_kver) \
DEFINE_BPF_PROG_EXT(SECTION_NAME, AID_ROOT, AID_ROOT, the_prog, min_kver, KVER_INF, \
BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, MANDATORY, \
"", "netd_readonly/", LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
DEFINE_NETD_RO_BPF_PROG("bind4/block_port", bind4_block_port, KVER_4_19)
(struct bpf_sock_addr *ctx) {
return block_port(ctx);
}
DEFINE_NETD_RO_BPF_PROG("bind6/block_port", bind6_block_port, KVER_4_19)
(struct bpf_sock_addr *ctx) {
return block_port(ctx);
}
LICENSE("Apache 2.0");
CRITICAL("ConnectivityNative");

163
bpf/progs/bpf_net_helpers.h Normal file
View file

@ -0,0 +1,163 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <linux/bpf.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/if_packet.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/pkt_cls.h>
#include <linux/tcp.h>
// bionic kernel uapi linux/udp.h header is munged...
#define __kernel_udphdr udphdr
#include <linux/udp.h>
#include <stdbool.h>
#include <stdint.h>
#include "bpf_helpers.h"
// IP flags. (from kernel's include/net/ip.h)
#define IP_CE 0x8000 // Flag: "Congestion" (really reserved 'evil bit')
#define IP_DF 0x4000 // Flag: "Don't Fragment"
#define IP_MF 0x2000 // Flag: "More Fragments"
#define IP_OFFSET 0x1FFF // "Fragment Offset" part
// IPv6 fragmentation header. (from kernel's include/net/ipv6.h)
struct frag_hdr {
__u8 nexthdr;
__u8 reserved; // always zero
__be16 frag_off; // 13 bit offset, 2 bits zero, 1 bit "More Fragments"
__be32 identification;
};
// ----- Helper functions for offsets to fields -----
// They all assume simple IP packets:
// - no VLAN ethernet tags
// - no IPv4 options (see IPV4_HLEN/TCP4_OFFSET/UDP4_OFFSET)
// - no IPv6 extension headers
// - no TCP options (see TCP_HLEN)
//#define ETH_HLEN sizeof(struct ethhdr)
#define IP4_HLEN sizeof(struct iphdr)
#define IP6_HLEN sizeof(struct ipv6hdr)
#define TCP_HLEN sizeof(struct tcphdr)
#define UDP_HLEN sizeof(struct udphdr)
// Offsets from beginning of L4 (TCP/UDP) header
#define TCP_OFFSET(field) offsetof(struct tcphdr, field)
#define UDP_OFFSET(field) offsetof(struct udphdr, field)
// Offsets from beginning of L3 (IPv4) header
#define IP4_OFFSET(field) offsetof(struct iphdr, field)
#define IP4_TCP_OFFSET(field) (IP4_HLEN + TCP_OFFSET(field))
#define IP4_UDP_OFFSET(field) (IP4_HLEN + UDP_OFFSET(field))
// Offsets from beginning of L3 (IPv6) header
#define IP6_OFFSET(field) offsetof(struct ipv6hdr, field)
#define IP6_TCP_OFFSET(field) (IP6_HLEN + TCP_OFFSET(field))
#define IP6_UDP_OFFSET(field) (IP6_HLEN + UDP_OFFSET(field))
// Offsets from beginning of L2 (ie. Ethernet) header (which must be present)
#define ETH_IP4_OFFSET(field) (ETH_HLEN + IP4_OFFSET(field))
#define ETH_IP4_TCP_OFFSET(field) (ETH_HLEN + IP4_TCP_OFFSET(field))
#define ETH_IP4_UDP_OFFSET(field) (ETH_HLEN + IP4_UDP_OFFSET(field))
#define ETH_IP6_OFFSET(field) (ETH_HLEN + IP6_OFFSET(field))
#define ETH_IP6_TCP_OFFSET(field) (ETH_HLEN + IP6_TCP_OFFSET(field))
#define ETH_IP6_UDP_OFFSET(field) (ETH_HLEN + IP6_UDP_OFFSET(field))
// this returns 0 iff skb->sk is NULL
static uint64_t (*bpf_get_socket_cookie)(struct __sk_buff* skb) = (void*)BPF_FUNC_get_socket_cookie;
static uint64_t (*bpf_get_sk_cookie)(struct bpf_sock* sk) = (void*)BPF_FUNC_get_socket_cookie;
static uint32_t (*bpf_get_socket_uid)(struct __sk_buff* skb) = (void*)BPF_FUNC_get_socket_uid;
static int (*bpf_skb_pull_data)(struct __sk_buff* skb, __u32 len) = (void*)BPF_FUNC_skb_pull_data;
static int (*bpf_skb_load_bytes)(const struct __sk_buff* skb, int off, void* to,
int len) = (void*)BPF_FUNC_skb_load_bytes;
static int (*bpf_skb_load_bytes_relative)(const struct __sk_buff* skb, int off, void* to, int len,
int start_hdr) = (void*)BPF_FUNC_skb_load_bytes_relative;
static int (*bpf_skb_store_bytes)(struct __sk_buff* skb, __u32 offset, const void* from, __u32 len,
__u64 flags) = (void*)BPF_FUNC_skb_store_bytes;
static int64_t (*bpf_csum_diff)(__be32* from, __u32 from_size, __be32* to, __u32 to_size,
__wsum seed) = (void*)BPF_FUNC_csum_diff;
static int64_t (*bpf_csum_update)(struct __sk_buff* skb, __wsum csum) = (void*)BPF_FUNC_csum_update;
static int (*bpf_skb_change_proto)(struct __sk_buff* skb, __be16 proto,
__u64 flags) = (void*)BPF_FUNC_skb_change_proto;
static int (*bpf_l3_csum_replace)(struct __sk_buff* skb, __u32 offset, __u64 from, __u64 to,
__u64 flags) = (void*)BPF_FUNC_l3_csum_replace;
static int (*bpf_l4_csum_replace)(struct __sk_buff* skb, __u32 offset, __u64 from, __u64 to,
__u64 flags) = (void*)BPF_FUNC_l4_csum_replace;
static int (*bpf_redirect)(__u32 ifindex, __u64 flags) = (void*)BPF_FUNC_redirect;
static int (*bpf_redirect_map)(const struct bpf_map_def* map, __u32 key,
__u64 flags) = (void*)BPF_FUNC_redirect_map;
static int (*bpf_skb_change_head)(struct __sk_buff* skb, __u32 head_room,
__u64 flags) = (void*)BPF_FUNC_skb_change_head;
static int (*bpf_skb_adjust_room)(struct __sk_buff* skb, __s32 len_diff, __u32 mode,
__u64 flags) = (void*)BPF_FUNC_skb_adjust_room;
// Android only supports little endian architectures
#define htons(x) (__builtin_constant_p(x) ? ___constant_swab16(x) : __builtin_bswap16(x))
#define htonl(x) (__builtin_constant_p(x) ? ___constant_swab32(x) : __builtin_bswap32(x))
#define ntohs(x) htons(x)
#define ntohl(x) htonl(x)
static inline __always_inline __unused bool is_received_skb(struct __sk_buff* skb) {
return skb->pkt_type == PACKET_HOST || skb->pkt_type == PACKET_BROADCAST ||
skb->pkt_type == PACKET_MULTICAST;
}
// try to make the first 'len' header bytes readable/writable via direct packet access
// (note: AFAIK there is no way to ask for only direct packet read without also getting write)
static inline __always_inline void try_make_writable(struct __sk_buff* skb, unsigned len) {
if (len > skb->len) len = skb->len;
if (skb->data_end - skb->data < len) bpf_skb_pull_data(skb, len);
}
struct egress_bool { bool egress; };
#define INGRESS ((struct egress_bool){ .egress = false })
#define EGRESS ((struct egress_bool){ .egress = true })
struct stream_bool { bool down; };
#define UPSTREAM ((struct stream_bool){ .down = false })
#define DOWNSTREAM ((struct stream_bool){ .down = true })
struct rawip_bool { bool rawip; };
#define ETHER ((struct rawip_bool){ .rawip = false })
#define RAWIP ((struct rawip_bool){ .rawip = true })
struct updatetime_bool { bool updatetime; };
#define NO_UPDATETIME ((struct updatetime_bool){ .updatetime = false })
#define UPDATETIME ((struct updatetime_bool){ .updatetime = true })
// Return value for xt_bpf (netfilter match extension) programs
static const int XTBPF_NOMATCH = 0;
static const int XTBPF_MATCH = 1;
static const int BPF_DISALLOW = 0;
static const int BPF_ALLOW = 1;

33
bpf/progs/clat_mark.h Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
/* -=-=-=-=-= WARNING -=-=-=-=-=-
*
* DO *NOT* *EVER* CHANGE THIS CONSTANT
*
* This is aidl::android::net::INetd::CLAT_MARK but we can't use that from
* pure C code (ie. the eBPF clat program).
*
* It must match the iptables rules setup by netd on Android T.
*
* This mark value is used by the eBPF clatd program to mark ingress non-offloaded clat
* packets for later dropping in ip6tables bw_raw_PREROUTING.
* They need to be dropped *after* the clat daemon (via receive on an AF_PACKET socket)
* sees them and thus cannot be dropped from the bpf program itself.
*/
static const uint32_t CLAT_MARK = 0xDEADC1A7;

401
bpf/progs/clatd.c Normal file
View file

@ -0,0 +1,401 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// The resulting .o needs to load on Android T+
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#include "bpf_net_helpers.h"
#include "clatd.h"
#include "clat_mark.h"
DEFINE_BPF_MAP_GRW(clat_ingress6_map, HASH, ClatIngress6Key, ClatIngress6Value, 16, AID_SYSTEM)
static inline __always_inline int nat64(struct __sk_buff* skb,
const struct rawip_bool rawip,
const struct kver_uint kver) {
const bool is_ethernet = !rawip.rawip;
// Require ethernet dst mac address to be our unicast address.
if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;
// Must be meta-ethernet IPv6 frame
if (skb->protocol != htons(ETH_P_IPV6)) return TC_ACT_PIPE;
const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
// Not clear if this is actually necessary considering we use DPA (Direct Packet Access),
// but we need to make sure we can read the IPv6 header reliably so that we can set
// skb->mark = 0xDeadC1a7 for packets we fail to offload.
try_make_writable(skb, l2_header_size + sizeof(struct ipv6hdr));
void* data = (void*)(long)skb->data;
const void* data_end = (void*)(long)skb->data_end;
const struct ethhdr* const eth = is_ethernet ? data : NULL; // used iff is_ethernet
const struct ipv6hdr* const ip6 = is_ethernet ? (void*)(eth + 1) : data;
// Must have (ethernet and) ipv6 header
if (data + l2_header_size + sizeof(*ip6) > data_end) return TC_ACT_PIPE;
// Ethertype - if present - must be IPv6
if (is_ethernet && (eth->h_proto != htons(ETH_P_IPV6))) return TC_ACT_PIPE;
// IP version must be 6
if (ip6->version != 6) return TC_ACT_PIPE;
// Maximum IPv6 payload length that can be translated to IPv4
// Note: technically this check is too strict for an IPv6 fragment,
// which by virtue of stripping the extra 8 byte fragment extension header,
// could thus be 8 bytes larger and still fit in an ipv4 packet post
// translation. However... who ever heard of receiving ~64KB frags...
// fragments are kind of by definition smaller than ingress device mtu,
// and thus, on the internet, very very unlikely to exceed 1500 bytes.
if (ntohs(ip6->payload_len) > 0xFFFF - sizeof(struct iphdr)) return TC_ACT_PIPE;
ClatIngress6Key k = {
.iif = skb->ifindex,
.pfx96.in6_u.u6_addr32 =
{
ip6->saddr.in6_u.u6_addr32[0],
ip6->saddr.in6_u.u6_addr32[1],
ip6->saddr.in6_u.u6_addr32[2],
},
.local6 = ip6->daddr,
};
ClatIngress6Value* v = bpf_clat_ingress6_map_lookup_elem(&k);
if (!v) return TC_ACT_PIPE;
__u8 proto = ip6->nexthdr;
__be16 ip_id = 0;
__be16 frag_off = htons(IP_DF);
__u16 tot_len = ntohs(ip6->payload_len) + sizeof(struct iphdr); // cannot overflow, see above
if (proto == IPPROTO_FRAGMENT) {
// Fragment handling requires bpf_skb_adjust_room which is 4.14+
if (!KVER_IS_AT_LEAST(kver, 4, 14, 0)) return TC_ACT_PIPE;
// Must have (ethernet and) ipv6 header and ipv6 fragment extension header
if (data + l2_header_size + sizeof(*ip6) + sizeof(struct frag_hdr) > data_end)
return TC_ACT_PIPE;
const struct frag_hdr *frag = (const struct frag_hdr *)(ip6 + 1);
proto = frag->nexthdr;
// RFC6145: use bottom 16-bits of network endian 32-bit IPv6 ID field for 16-bit IPv4 field.
// this is equivalent to: ip_id = htons(ntohl(frag->identification));
ip_id = frag->identification >> 16;
// Conversion of 16-bit IPv6 frag offset to 16-bit IPv4 frag offset field.
// IPv6 is '13 bits of offset in multiples of 8' + 2 zero bits + more fragment bit
// IPv4 is zero bit + don't frag bit + more frag bit + '13 bits of offset in multiples of 8'
frag_off = ntohs(frag->frag_off);
frag_off = ((frag_off & 1) << 13) | (frag_off >> 3);
frag_off = htons(frag_off);
// Note that by construction tot_len is guaranteed to not underflow here
tot_len -= sizeof(struct frag_hdr);
// This is a badly formed IPv6 packet with less payload than the size of an IPv6 Frag EH
if (tot_len < sizeof(struct iphdr)) return TC_ACT_PIPE;
}
switch (proto) {
case IPPROTO_TCP: // For TCP, UDP & UDPLITE the checksum neutrality of the chosen
case IPPROTO_UDP: // IPv6 address means there is no need to update their checksums.
case IPPROTO_UDPLITE: //
case IPPROTO_GRE: // We do not need to bother looking at GRE/ESP headers,
case IPPROTO_ESP: // since there is never a checksum to update.
break;
default: // do not know how to handle anything else
// Mark ingress non-offloaded clat packet for dropping in ip6tables bw_raw_PREROUTING.
// Non-offloaded clat packet is going to be handled by clat daemon and ip6tables. The
// duplicate one in ip6tables is not necessary.
skb->mark = CLAT_MARK;
return TC_ACT_PIPE;
}
struct ethhdr eth2; // used iff is_ethernet
if (is_ethernet) {
eth2 = *eth; // Copy over the ethernet header (src/dst mac)
eth2.h_proto = htons(ETH_P_IP); // But replace the ethertype
}
struct iphdr ip = {
.version = 4, // u4
.ihl = sizeof(struct iphdr) / sizeof(__u32), // u4
.tos = (ip6->priority << 4) + (ip6->flow_lbl[0] >> 4), // u8
.tot_len = htons(tot_len), // be16
.id = ip_id, // be16
.frag_off = frag_off, // be16
.ttl = ip6->hop_limit, // u8
.protocol = proto, // u8
.check = 0, // u16
.saddr = ip6->saddr.in6_u.u6_addr32[3], // be32
.daddr = v->local4.s_addr, // be32
};
// Calculate the IPv4 one's complement checksum of the IPv4 header.
__wsum sum4 = 0;
for (unsigned i = 0; i < sizeof(ip) / sizeof(__u16); ++i) {
sum4 += ((__u16*)&ip)[i];
}
// Note that sum4 is guaranteed to be non-zero by virtue of ip.version == 4
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse u32 into range 1 .. 0x1FFFE
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse any potential carry into u16
ip.check = (__u16)~sum4; // sum4 cannot be zero, so this is never 0xFFFF
// Calculate the *negative* IPv6 16-bit one's complement checksum of the IPv6 header.
__wsum sum6 = 0;
// We'll end up with a non-zero sum due to ip6->version == 6 (which has '0' bits)
for (unsigned i = 0; i < sizeof(*ip6) / sizeof(__u16); ++i) {
sum6 += ~((__u16*)ip6)[i]; // note the bitwise negation
}
// Note that there is no L4 checksum update: we are relying on the checksum neutrality
// of the ipv6 address chosen by netd's ClatdController.
// Packet mutations begin - point of no return, but if this first modification fails
// the packet is probably still pristine, so let clatd handle it.
if (bpf_skb_change_proto(skb, htons(ETH_P_IP), 0)) {
// Mark ingress non-offloaded clat packet for dropping in ip6tables bw_raw_PREROUTING.
// Non-offloaded clat packet is going to be handled by clat daemon and ip6tables. The
// duplicate one in ip6tables is not necessary.
skb->mark = CLAT_MARK;
return TC_ACT_PIPE;
}
// This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
//
// In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
// thus we need to subtract out the ipv6 header's sum, and add in the ipv4 header's sum.
// However, by construction of ip.check above the checksum of an ipv4 header is zero.
// Thus we only need to subtract the ipv6 header's sum, which is the same as adding
// in the sum of the bitwise negation of the ipv6 header.
//
// bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
// (-ENOTSUPP) if it isn't. So we just ignore the return code.
//
// if (skb->ip_summed == CHECKSUM_COMPLETE)
// return (skb->csum = csum_add(skb->csum, csum));
// else
// return -ENOTSUPP;
bpf_csum_update(skb, sum6);
// Technically 'kver < KVER_4_14' already implies 'frag_off == htons(IP_DF)' due to logic above,
// thus the initial 'kver >= KVER_4_14' check here is entirely superfluous.
//
// However, we *need* the compiler (when compiling the program for 4.9) to entirely
// optimize out the call to bpf_skb_adjust_room() bpf helper: it's not enough for it to emit
// an unreachable call to it, it must *not* emit it at all (otherwise the 4.9 kernel's
// bpf verifier will refuse to load a program with an unknown bpf helper call)
//
// This is easiest to achieve by being very explicit in the if clause,
// better safe than sorry...
//
// Note: we currently have no TreeHugger coverage for 4.9-T devices (there are no such
// Pixel or cuttlefish devices), so likely you won't notice for months if this breaks...
if (KVER_IS_AT_LEAST(kver, 4, 14, 0) && frag_off != htons(IP_DF)) {
// If we're converting an IPv6 Fragment, we need to trim off 8 more bytes
// We're beyond recovery on error here... but hard to imagine how this could fail.
if (bpf_skb_adjust_room(skb, -(__s32)sizeof(struct frag_hdr), BPF_ADJ_ROOM_NET, /*flags*/0))
return TC_ACT_SHOT;
}
try_make_writable(skb, l2_header_size + sizeof(struct iphdr));
// bpf_skb_change_proto() invalidates all pointers - reload them.
data = (void*)(long)skb->data;
data_end = (void*)(long)skb->data_end;
// I cannot think of any valid way for this error condition to trigger, however I do
// believe the explicit check is required to keep the in kernel ebpf verifier happy.
if (data + l2_header_size + sizeof(struct iphdr) > data_end) return TC_ACT_SHOT;
if (is_ethernet) {
struct ethhdr* new_eth = data;
// Copy over the updated ethernet header
*new_eth = eth2;
// Copy over the new ipv4 header.
*(struct iphdr*)(new_eth + 1) = ip;
} else {
// Copy over the new ipv4 header without an ethernet header.
*(struct iphdr*)data = ip;
}
// Count successfully translated packet
__sync_fetch_and_add(&v->packets, 1);
__sync_fetch_and_add(&v->bytes, skb->len - l2_header_size);
// Redirect, possibly back to same interface, so tcpdump sees packet twice.
if (v->oif) return bpf_redirect(v->oif, BPF_F_INGRESS);
// Just let it through, tcpdump will not see IPv4 packet.
return TC_ACT_PIPE;
}
DEFINE_BPF_PROG_KVER("schedcls/ingress6/clat_ether$4_14", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_ether_4_14, KVER_4_14)
(struct __sk_buff* skb) {
return nat64(skb, ETHER, KVER_4_14);
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/ingress6/clat_ether$4_9", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_ether_4_9, KVER_NONE, KVER_4_14)
(struct __sk_buff* skb) {
return nat64(skb, ETHER, KVER_NONE);
}
DEFINE_BPF_PROG_KVER("schedcls/ingress6/clat_rawip$4_14", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_rawip_4_14, KVER_4_14)
(struct __sk_buff* skb) {
return nat64(skb, RAWIP, KVER_4_14);
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/ingress6/clat_rawip$4_9", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_rawip_4_9, KVER_NONE, KVER_4_14)
(struct __sk_buff* skb) {
return nat64(skb, RAWIP, KVER_NONE);
}
DEFINE_BPF_MAP_GRW(clat_egress4_map, HASH, ClatEgress4Key, ClatEgress4Value, 16, AID_SYSTEM)
DEFINE_BPF_PROG("schedcls/egress4/clat_rawip", AID_ROOT, AID_SYSTEM, sched_cls_egress4_clat_rawip)
(struct __sk_buff* skb) {
// Must be meta-ethernet IPv4 frame
if (skb->protocol != htons(ETH_P_IP)) return TC_ACT_PIPE;
// Possibly not needed, but for consistency with nat64 up above
try_make_writable(skb, sizeof(struct iphdr));
void* data = (void*)(long)skb->data;
const void* data_end = (void*)(long)skb->data_end;
const struct iphdr* const ip4 = data;
// Must have ipv4 header
if (data + sizeof(*ip4) > data_end) return TC_ACT_PIPE;
// IP version must be 4
if (ip4->version != 4) return TC_ACT_PIPE;
// We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
if (ip4->ihl != 5) return TC_ACT_PIPE;
// Calculate the IPv4 one's complement checksum of the IPv4 header.
__wsum sum4 = 0;
for (unsigned i = 0; i < sizeof(*ip4) / sizeof(__u16); ++i) {
sum4 += ((__u16*)ip4)[i];
}
// Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse u32 into range 1 .. 0x1FFFE
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse any potential carry into u16
// for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
if (sum4 != 0xFFFF) return TC_ACT_PIPE;
// Minimum IPv4 total length is the size of the header
if (ntohs(ip4->tot_len) < sizeof(*ip4)) return TC_ACT_PIPE;
// We are incapable of dealing with IPv4 fragments
if (ip4->frag_off & ~htons(IP_DF)) return TC_ACT_PIPE;
switch (ip4->protocol) {
case IPPROTO_TCP: // For TCP, UDP & UDPLITE the checksum neutrality of the chosen
case IPPROTO_UDPLITE: // IPv6 address means there is no need to update their checksums.
case IPPROTO_GRE: // We do not need to bother looking at GRE/ESP headers,
case IPPROTO_ESP: // since there is never a checksum to update.
break;
case IPPROTO_UDP: // See above comment, but must also have UDP header...
if (data + sizeof(*ip4) + sizeof(struct udphdr) > data_end) return TC_ACT_PIPE;
const struct udphdr* uh = (const struct udphdr*)(ip4 + 1);
// If IPv4/UDP checksum is 0 then fallback to clatd so it can calculate the
// checksum. Otherwise the network or more likely the NAT64 gateway might
// drop the packet because in most cases IPv6/UDP packets with a zero checksum
// are invalid. See RFC 6935. TODO: calculate checksum via bpf_csum_diff()
if (!uh->check) return TC_ACT_PIPE;
break;
default: // do not know how to handle anything else
return TC_ACT_PIPE;
}
ClatEgress4Key k = {
.iif = skb->ifindex,
.local4.s_addr = ip4->saddr,
};
ClatEgress4Value* v = bpf_clat_egress4_map_lookup_elem(&k);
if (!v) return TC_ACT_PIPE;
// Translating without redirecting doesn't make sense.
if (!v->oif) return TC_ACT_PIPE;
// This implementation is currently limited to rawip.
if (v->oifIsEthernet) return TC_ACT_PIPE;
struct ipv6hdr ip6 = {
.version = 6, // __u8:4
.priority = ip4->tos >> 4, // __u8:4
.flow_lbl = {(ip4->tos & 0xF) << 4, 0, 0}, // __u8[3]
.payload_len = htons(ntohs(ip4->tot_len) - 20), // __be16
.nexthdr = ip4->protocol, // __u8
.hop_limit = ip4->ttl, // __u8
.saddr = v->local6, // struct in6_addr
.daddr = v->pfx96, // struct in6_addr
};
ip6.daddr.in6_u.u6_addr32[3] = ip4->daddr;
// Calculate the IPv6 16-bit one's complement checksum of the IPv6 header.
__wsum sum6 = 0;
// We'll end up with a non-zero sum due to ip6.version == 6
for (unsigned i = 0; i < sizeof(ip6) / sizeof(__u16); ++i) {
sum6 += ((__u16*)&ip6)[i];
}
// Note that there is no L4 checksum update: we are relying on the checksum neutrality
// of the ipv6 address chosen by netd's ClatdController.
// Packet mutations begin - point of no return, but if this first modification fails
// the packet is probably still pristine, so let clatd handle it.
if (bpf_skb_change_proto(skb, htons(ETH_P_IPV6), 0)) return TC_ACT_PIPE;
// This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
//
// In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
// thus we need to subtract out the ipv4 header's sum, and add in the ipv6 header's sum.
// However, we've already verified the ipv4 checksum is correct and thus 0.
// Thus we only need to add the ipv6 header's sum.
//
// bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
// (-ENOTSUPP) if it isn't. So we just ignore the return code (see above for more details).
bpf_csum_update(skb, sum6);
// bpf_skb_change_proto() invalidates all pointers - reload them.
data = (void*)(long)skb->data;
data_end = (void*)(long)skb->data_end;
// I cannot think of any valid way for this error condition to trigger, however I do
// believe the explicit check is required to keep the in kernel ebpf verifier happy.
if (data + sizeof(ip6) > data_end) return TC_ACT_SHOT;
// Copy over the new ipv6 header without an ethernet header.
*(struct ipv6hdr*)data = ip6;
// Count successfully translated packet
__sync_fetch_and_add(&v->packets, 1);
__sync_fetch_and_add(&v->bytes, skb->len);
// Redirect to non v4-* interface. Tcpdump only sees packet after this redirect.
return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
}
LICENSE("Apache 2.0");
CRITICAL("Connectivity");

64
bpf/progs/clatd.h Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <linux/in.h>
#include <linux/in6.h>
#include <stdbool.h>
#include <stdint.h>
// This header file is shared by eBPF kernel programs (C) and netd (C++) and
// some of the maps are also accessed directly from Java mainline module code.
//
// Hence: explicitly pad all relevant structures and assert that their size
// is the sum of the sizes of their fields.
#define STRUCT_SIZE(name, size) _Static_assert(sizeof(name) == (size), "Incorrect struct size.")
typedef struct {
uint32_t iif; // The input interface index
struct in6_addr pfx96; // The source /96 nat64 prefix, bottom 32 bits must be 0
struct in6_addr local6; // The full 128-bits of the destination IPv6 address
} ClatIngress6Key;
STRUCT_SIZE(ClatIngress6Key, 4 + 2 * 16); // 36
typedef struct {
uint32_t oif; // The output interface to redirect to (0 means don't redirect)
struct in_addr local4; // The destination IPv4 address
uint64_t packets; // Count of translated gso (large) packets
uint64_t bytes; // Sum of post-translation skb->len
} ClatIngress6Value;
STRUCT_SIZE(ClatIngress6Value, 4 + 4 + 8 + 8); // 24
typedef struct {
uint32_t iif; // The input interface index
struct in_addr local4; // The source IPv4 address
} ClatEgress4Key;
STRUCT_SIZE(ClatEgress4Key, 4 + 4); // 8
typedef struct {
uint32_t oif; // The output interface to redirect to
struct in6_addr local6; // The full 128-bits of the source IPv6 address
struct in6_addr pfx96; // The destination /96 nat64 prefix, bottom 32 bits must be 0
bool oifIsEthernet; // Whether the output interface requires ethernet header
uint8_t pad[3];
uint64_t packets; // Count of translated gso (large) packets
uint64_t bytes; // Sum of post-translation skb->len
} ClatEgress4Value;
STRUCT_SIZE(ClatEgress4Value, 4 + 2 * 16 + 1 + 3 + 8 + 8); // 56
#undef STRUCT_SIZE

226
bpf/progs/dscpPolicy.c Normal file
View file

@ -0,0 +1,226 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// The resulting .o needs to load on Android T+
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#include "bpf_net_helpers.h"
#include "dscpPolicy.h"
#define ECN_MASK 3
#define UPDATE_TOS(dscp, tos) ((dscp) << 2) | ((tos) & ECN_MASK)
DEFINE_BPF_MAP_GRW(socket_policy_cache_map, HASH, uint64_t, RuleEntry, CACHE_MAP_SIZE, AID_SYSTEM)
DEFINE_BPF_MAP_GRW(ipv4_dscp_policies_map, ARRAY, uint32_t, DscpPolicy, MAX_POLICIES, AID_SYSTEM)
DEFINE_BPF_MAP_GRW(ipv6_dscp_policies_map, ARRAY, uint32_t, DscpPolicy, MAX_POLICIES, AID_SYSTEM)
static inline __always_inline void match_policy(struct __sk_buff* skb, bool ipv4) {
void* data = (void*)(long)skb->data;
const void* data_end = (void*)(long)skb->data_end;
const int l2_header_size = sizeof(struct ethhdr);
struct ethhdr* eth = data;
if (data + l2_header_size > data_end) return;
int hdr_size = 0;
// used for map lookup
uint64_t cookie = bpf_get_socket_cookie(skb);
if (!cookie) return;
__be16 sport = 0;
uint16_t dport = 0;
uint8_t protocol = 0; // TODO: Use are reserved value? Or int (-1) and cast to uint below?
struct in6_addr src_ip = {};
struct in6_addr dst_ip = {};
uint8_t tos = 0; // Only used for IPv4
__be32 old_first_be32 = 0; // Only used for IPv6
if (ipv4) {
const struct iphdr* const iph = (void*)(eth + 1);
hdr_size = l2_header_size + sizeof(struct iphdr);
// Must have ipv4 header
if (data + hdr_size > data_end) return;
// IP version must be 4
if (iph->version != 4) return;
// We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
if (iph->ihl != 5) return;
// V4 mapped address in in6_addr sets 10/11 position to 0xff.
src_ip.s6_addr32[2] = htonl(0x0000ffff);
dst_ip.s6_addr32[2] = htonl(0x0000ffff);
// Copy IPv4 address into in6_addr for easy comparison below.
src_ip.s6_addr32[3] = iph->saddr;
dst_ip.s6_addr32[3] = iph->daddr;
protocol = iph->protocol;
tos = iph->tos;
} else {
struct ipv6hdr* ip6h = (void*)(eth + 1);
hdr_size = l2_header_size + sizeof(struct ipv6hdr);
// Must have ipv6 header
if (data + hdr_size > data_end) return;
if (ip6h->version != 6) return;
src_ip = ip6h->saddr;
dst_ip = ip6h->daddr;
protocol = ip6h->nexthdr;
old_first_be32 = *(__be32*)ip6h;
}
switch (protocol) {
case IPPROTO_UDP:
case IPPROTO_UDPLITE: {
struct udphdr* udp;
udp = data + hdr_size;
if ((void*)(udp + 1) > data_end) return;
sport = udp->source;
dport = ntohs(udp->dest);
} break;
case IPPROTO_TCP: {
struct tcphdr* tcp;
tcp = data + hdr_size;
if ((void*)(tcp + 1) > data_end) return;
sport = tcp->source;
dport = ntohs(tcp->dest);
} break;
default:
return;
}
RuleEntry* existing_rule = bpf_socket_policy_cache_map_lookup_elem(&cookie);
if (existing_rule &&
v6_equal(src_ip, existing_rule->src_ip) &&
v6_equal(dst_ip, existing_rule->dst_ip) &&
skb->ifindex == existing_rule->ifindex &&
sport == existing_rule->src_port &&
dport == existing_rule->dst_port &&
protocol == existing_rule->proto) {
if (existing_rule->dscp_val < 0) return;
if (ipv4) {
uint8_t newTos = UPDATE_TOS(existing_rule->dscp_val, tos);
bpf_l3_csum_replace(skb, l2_header_size + IP4_OFFSET(check), htons(tos), htons(newTos),
sizeof(uint16_t));
bpf_skb_store_bytes(skb, l2_header_size + IP4_OFFSET(tos), &newTos, sizeof(newTos), 0);
} else {
__be32 new_first_be32 =
htonl(ntohl(old_first_be32) & 0xF03FFFFF | (existing_rule->dscp_val << 22));
bpf_skb_store_bytes(skb, l2_header_size, &new_first_be32, sizeof(__be32),
BPF_F_RECOMPUTE_CSUM);
}
return;
}
// Linear scan ipv4_dscp_policies_map since no stored params match skb.
int best_score = 0;
int8_t new_dscp = -1;
for (register uint64_t i = 0; i < MAX_POLICIES; i++) {
// Using a uint64 in for loop prevents infinite loop during BPF load,
// but the key is uint32, so convert back.
uint32_t key = i;
DscpPolicy* policy;
if (ipv4) {
policy = bpf_ipv4_dscp_policies_map_lookup_elem(&key);
} else {
policy = bpf_ipv6_dscp_policies_map_lookup_elem(&key);
}
// If the policy lookup failed, just continue (this should not ever happen)
if (!policy) continue;
// If policy iface index does not match skb, then skip to next policy.
if (policy->ifindex != skb->ifindex) continue;
int score = 0;
if (policy->present_fields & PROTO_MASK_FLAG) {
if (protocol != policy->proto) continue;
score += 0xFFFF;
}
if (policy->present_fields & SRC_IP_MASK_FLAG) {
if (v6_not_equal(src_ip, policy->src_ip)) continue;
score += 0xFFFF;
}
if (policy->present_fields & DST_IP_MASK_FLAG) {
if (v6_not_equal(dst_ip, policy->dst_ip)) continue;
score += 0xFFFF;
}
if (policy->present_fields & SRC_PORT_MASK_FLAG) {
if (sport != policy->src_port) continue;
score += 0xFFFF;
}
if (dport < policy->dst_port_start) continue;
if (dport > policy->dst_port_end) continue;
score += 0xFFFF + policy->dst_port_start - policy->dst_port_end;
if (score > best_score) {
best_score = score;
new_dscp = policy->dscp_val;
}
}
RuleEntry value = {
.src_ip = src_ip,
.dst_ip = dst_ip,
.ifindex = skb->ifindex,
.src_port = sport,
.dst_port = dport,
.proto = protocol,
.dscp_val = new_dscp,
};
// Update cache with found policy.
bpf_socket_policy_cache_map_update_elem(&cookie, &value, BPF_ANY);
if (new_dscp < 0) return;
// Need to store bytes after updating map or program will not load.
if (ipv4) {
uint8_t new_tos = UPDATE_TOS(new_dscp, tos);
bpf_l3_csum_replace(skb, l2_header_size + IP4_OFFSET(check), htons(tos), htons(new_tos), 2);
bpf_skb_store_bytes(skb, l2_header_size + IP4_OFFSET(tos), &new_tos, sizeof(new_tos), 0);
} else {
__be32 new_first_be32 = htonl(ntohl(old_first_be32) & 0xF03FFFFF | (new_dscp << 22));
bpf_skb_store_bytes(skb, l2_header_size, &new_first_be32, sizeof(__be32),
BPF_F_RECOMPUTE_CSUM);
}
return;
}
DEFINE_BPF_PROG_KVER("schedcls/set_dscp_ether", AID_ROOT, AID_SYSTEM, schedcls_set_dscp_ether,
KVER_5_15)
(struct __sk_buff* skb) {
if (skb->pkt_type != PACKET_HOST) return TC_ACT_PIPE;
if (skb->protocol == htons(ETH_P_IP)) {
match_policy(skb, true);
} else if (skb->protocol == htons(ETH_P_IPV6)) {
match_policy(skb, false);
}
// Always return TC_ACT_PIPE
return TC_ACT_PIPE;
}
LICENSE("Apache 2.0");
CRITICAL("Connectivity");

64
bpf/progs/dscpPolicy.h Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright (C) 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#define CACHE_MAP_SIZE 1024
#define MAX_POLICIES 16
#define SRC_IP_MASK_FLAG 1
#define DST_IP_MASK_FLAG 2
#define SRC_PORT_MASK_FLAG 4
#define PROTO_MASK_FLAG 8
#define STRUCT_SIZE(name, size) _Static_assert(sizeof(name) == (size), "Incorrect struct size.")
// Retrieve the first (ie. high) 64 bits of an IPv6 address (in network order)
#define v6_hi_be64(v) (*(uint64_t*)&((v).s6_addr32[0]))
// Retrieve the last (ie. low) 64 bits of an IPv6 address (in network order)
#define v6_lo_be64(v) (*(uint64_t*)&((v).s6_addr32[2]))
// This returns a non-zero u64 iff a != b
#define v6_not_equal(a, b) ((v6_hi_be64(a) ^ v6_hi_be64(b)) \
| (v6_lo_be64(a) ^ v6_lo_be64(b)))
// Returns 'a == b' as boolean
#define v6_equal(a, b) (!v6_not_equal((a), (b)))
typedef struct {
struct in6_addr src_ip;
struct in6_addr dst_ip;
uint32_t ifindex;
__be16 src_port;
uint16_t dst_port_start;
uint16_t dst_port_end;
uint8_t proto;
int8_t dscp_val; // -1 none, or 0..63 DSCP value
uint8_t present_fields;
uint8_t pad[3];
} DscpPolicy;
STRUCT_SIZE(DscpPolicy, 2 * 16 + 4 + 3 * 2 + 3 * 1 + 3); // 48
typedef struct {
struct in6_addr src_ip;
struct in6_addr dst_ip;
uint32_t ifindex;
__be16 src_port;
uint16_t dst_port;
uint8_t proto;
int8_t dscp_val; // -1 none, or 0..63 DSCP value
uint8_t pad[2];
} RuleEntry;
STRUCT_SIZE(RuleEntry, 2 * 16 + 1 * 4 + 2 * 2 + 2 * 1 + 2); // 44

720
bpf/progs/netd.c Normal file
View file

@ -0,0 +1,720 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// The resulting .o needs to load on Android T+
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#include "bpf_net_helpers.h"
#include "netd.h"
// This is defined for cgroup bpf filter only.
static const int DROP = 0;
static const int PASS = 1;
static const int DROP_UNLESS_DNS = 2; // internal to our program
// Used for 'bool enable_tracing'
static const bool TRACE_ON = true;
static const bool TRACE_OFF = false;
// offsetof(struct iphdr, ihl) -- but that's a bitfield
#define IPPROTO_IHL_OFF 0
// This is offsetof(struct tcphdr, "32 bit tcp flag field")
// The tcp flags are after be16 source, dest & be32 seq, ack_seq, hence 12 bytes in.
//
// Note that TCP_FLAG_{ACK,PSH,RST,SYN,FIN} are htonl(0x00{10,08,04,02,01}0000)
// see include/uapi/linux/tcp.h
#define TCP_FLAG32_OFF 12
#define TCP_FLAG8_OFF (TCP_FLAG32_OFF + 1)
// For maps netd does not need to access
#define DEFINE_BPF_MAP_NO_NETD(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries) \
DEFINE_BPF_MAP_EXT(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries, \
AID_ROOT, AID_NET_BW_ACCT, 0060, "fs_bpf_net_shared", "", \
PRIVATE, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, \
LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
// For maps netd only needs read only access to
#define DEFINE_BPF_MAP_RO_NETD(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries) \
DEFINE_BPF_MAP_EXT(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries, \
AID_ROOT, AID_NET_BW_ACCT, 0460, "fs_bpf_netd_readonly", "", \
PRIVATE, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, \
LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
// For maps netd needs to be able to read and write
#define DEFINE_BPF_MAP_RW_NETD(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries) \
DEFINE_BPF_MAP_UGM(the_map, TYPE, TypeOfKey, TypeOfValue, num_entries, \
AID_ROOT, AID_NET_BW_ACCT, 0660)
// Bpf map arrays on creation are preinitialized to 0 and do not support deletion of a key,
// see: kernel/bpf/arraymap.c array_map_delete_elem() returns -EINVAL (from both syscall and ebpf)
// Additionally on newer kernels the bpf jit can optimize out the lookups.
// only valid indexes are [0..CONFIGURATION_MAP_SIZE-1]
DEFINE_BPF_MAP_RO_NETD(configuration_map, ARRAY, uint32_t, uint32_t, CONFIGURATION_MAP_SIZE)
// TODO: consider whether we can merge some of these maps
// for example it might be possible to merge 2 or 3 of:
// uid_counterset_map + uid_owner_map + uid_permission_map
DEFINE_BPF_MAP_RW_NETD(cookie_tag_map, HASH, uint64_t, UidTagValue, COOKIE_UID_MAP_SIZE)
DEFINE_BPF_MAP_NO_NETD(uid_counterset_map, HASH, uint32_t, uint8_t, UID_COUNTERSET_MAP_SIZE)
DEFINE_BPF_MAP_NO_NETD(app_uid_stats_map, HASH, uint32_t, StatsValue, APP_STATS_MAP_SIZE)
DEFINE_BPF_MAP_RO_NETD(stats_map_A, HASH, StatsKey, StatsValue, STATS_MAP_SIZE)
DEFINE_BPF_MAP_RO_NETD(stats_map_B, HASH, StatsKey, StatsValue, STATS_MAP_SIZE)
DEFINE_BPF_MAP_NO_NETD(iface_stats_map, HASH, uint32_t, StatsValue, IFACE_STATS_MAP_SIZE)
DEFINE_BPF_MAP_RO_NETD(uid_owner_map, HASH, uint32_t, UidOwnerValue, UID_OWNER_MAP_SIZE)
DEFINE_BPF_MAP_RO_NETD(uid_permission_map, HASH, uint32_t, uint8_t, UID_OWNER_MAP_SIZE)
DEFINE_BPF_MAP_NO_NETD(ingress_discard_map, HASH, IngressDiscardKey, IngressDiscardValue,
INGRESS_DISCARD_MAP_SIZE)
DEFINE_BPF_MAP_RW_NETD(lock_array_test_map, ARRAY, uint32_t, bool, 1)
DEFINE_BPF_MAP_RW_NETD(lock_hash_test_map, HASH, uint32_t, bool, 1)
/* never actually used from ebpf */
DEFINE_BPF_MAP_NO_NETD(iface_index_name_map, HASH, uint32_t, IfaceValue, IFACE_INDEX_NAME_MAP_SIZE)
// A single-element configuration array, packet tracing is enabled when 'true'.
DEFINE_BPF_MAP_EXT(packet_trace_enabled_map, ARRAY, uint32_t, bool, 1,
AID_ROOT, AID_SYSTEM, 0060, "fs_bpf_net_shared", "", PRIVATE,
BPFLOADER_MAINLINE_U_VERSION, BPFLOADER_MAX_VER, LOAD_ON_ENG,
LOAD_ON_USER, LOAD_ON_USERDEBUG)
// A ring buffer on which packet information is pushed.
DEFINE_BPF_RINGBUF_EXT(packet_trace_ringbuf, PacketTrace, PACKET_TRACE_BUF_SIZE,
AID_ROOT, AID_SYSTEM, 0060, "fs_bpf_net_shared", "", PRIVATE,
BPFLOADER_MAINLINE_U_VERSION, BPFLOADER_MAX_VER, LOAD_ON_ENG,
LOAD_ON_USER, LOAD_ON_USERDEBUG);
DEFINE_BPF_MAP_RO_NETD(data_saver_enabled_map, ARRAY, uint32_t, bool,
DATA_SAVER_ENABLED_MAP_SIZE)
// iptables xt_bpf programs need to be usable by both netd and netutils_wrappers
// selinux contexts, because even non-xt_bpf iptables mutations are implemented as
// a full table dump, followed by an update in userspace, and then a reload into the kernel,
// where any already in-use xt_bpf matchers are serialized as the path to the pinned
// program (see XT_BPF_MODE_PATH_PINNED) and then the iptables binary (or rather
// the kernel acting on behalf of it) must be able to retrieve the pinned program
// for the reload to succeed
#define DEFINE_XTBPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \
DEFINE_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog)
// programs that need to be usable by netd, but not by netutils_wrappers
// (this is because these are currently attached by the mainline provided libnetd_updatable .so
// which is loaded into netd and thus runs as netd uid/gid/selinux context)
#define DEFINE_NETD_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, minKV, maxKV) \
DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, \
minKV, maxKV, BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, MANDATORY, \
"fs_bpf_netd_readonly", "", LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
#define DEFINE_NETD_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \
DEFINE_NETD_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF)
#define DEFINE_NETD_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \
DEFINE_NETD_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, KVER_NONE)
#define DEFINE_NETD_V_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, minKV) \
DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, minKV, \
KVER_INF, BPFLOADER_MAINLINE_V_VERSION, BPFLOADER_MAX_VER, MANDATORY, \
"fs_bpf_netd_readonly", "", LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
// programs that only need to be usable by the system server
#define DEFINE_SYS_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \
DEFINE_BPF_PROG_EXT(SECTION_NAME, prog_uid, prog_gid, the_prog, KVER_NONE, KVER_INF, \
BPFLOADER_MIN_VER, BPFLOADER_MAX_VER, MANDATORY, \
"fs_bpf_net_shared", "", LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
/*
* Note: this blindly assumes an MTU of 1500, and that packets > MTU are always TCP,
* and that TCP is using the Linux default settings with TCP timestamp option enabled
* which uses 12 TCP option bytes per frame.
*
* These are not unreasonable assumptions:
*
* The internet does not really support MTUs greater than 1500, so most TCP traffic will
* be at that MTU, or slightly below it (worst case our upwards adjustment is too small).
*
* The chance our traffic isn't IP at all is basically zero, so the IP overhead correction
* is bound to be needed.
*
* Furthermore, the likelyhood that we're having to deal with GSO (ie. > MTU) packets that
* are not IP/TCP is pretty small (few other things are supported by Linux) and worse case
* our extra overhead will be slightly off, but probably still better than assuming none.
*
* Most servers are also Linux and thus support/default to using TCP timestamp option
* (and indeed TCP timestamp option comes from RFC 1323 titled "TCP Extensions for High
* Performance" which also defined TCP window scaling and are thus absolutely ancient...).
*
* All together this should be more correct than if we simply ignored GSO frames
* (ie. counted them as single packets with no extra overhead)
*
* Especially since the number of packets is important for any future clat offload correction.
* (which adjusts upward by 20 bytes per packet to account for ipv4 -> ipv6 header conversion)
*/
#define DEFINE_UPDATE_STATS(the_stats_map, TypeOfKey) \
static __always_inline inline void update_##the_stats_map(const struct __sk_buff* const skb, \
const TypeOfKey* const key, \
const struct egress_bool egress, \
__unused const struct kver_uint kver) { \
StatsValue* value = bpf_##the_stats_map##_lookup_elem(key); \
if (!value) { \
StatsValue newValue = {}; \
bpf_##the_stats_map##_update_elem(key, &newValue, BPF_NOEXIST); \
value = bpf_##the_stats_map##_lookup_elem(key); \
} \
if (value) { \
const int mtu = 1500; \
uint64_t packets = 1; \
uint64_t bytes = skb->len; \
if (bytes > mtu) { \
bool is_ipv6 = (skb->protocol == htons(ETH_P_IPV6)); \
int ip_overhead = (is_ipv6 ? sizeof(struct ipv6hdr) : sizeof(struct iphdr)); \
int tcp_overhead = ip_overhead + sizeof(struct tcphdr) + 12; \
int mss = mtu - tcp_overhead; \
uint64_t payload = bytes - tcp_overhead; \
packets = (payload + mss - 1) / mss; \
bytes = tcp_overhead * packets + payload; \
} \
if (egress.egress) { \
__sync_fetch_and_add(&value->txPackets, packets); \
__sync_fetch_and_add(&value->txBytes, bytes); \
} else { \
__sync_fetch_and_add(&value->rxPackets, packets); \
__sync_fetch_and_add(&value->rxBytes, bytes); \
} \
} \
}
DEFINE_UPDATE_STATS(app_uid_stats_map, uint32_t)
DEFINE_UPDATE_STATS(iface_stats_map, uint32_t)
DEFINE_UPDATE_STATS(stats_map_A, StatsKey)
DEFINE_UPDATE_STATS(stats_map_B, StatsKey)
// both of these return 0 on success or -EFAULT on failure (and zero out the buffer)
static __always_inline inline int bpf_skb_load_bytes_net(const struct __sk_buff* const skb,
const int L3_off,
void* const to,
const int len,
const struct kver_uint kver) {
// 'kver' (here and throughout) is the compile time guaranteed minimum kernel version,
// ie. we're building (a version of) the bpf program for kver (or newer!) kernels.
//
// 4.19+ kernels support the 'bpf_skb_load_bytes_relative()' bpf helper function,
// so we can use it. On pre-4.19 kernels we cannot use the relative load helper,
// and thus will simply get things wrong if there's any L2 (ethernet) header in the skb.
//
// Luckily, for cellular traffic, there likely isn't any, as cell is usually 'rawip'.
//
// However, this does mean that wifi (and ethernet) on 4.14 is basically a lost cause:
// we'll be making decisions based on the *wrong* bytes (fetched from the wrong offset),
// because the 'L3_off' passed to bpf_skb_load_bytes() should be increased by l2_header_size,
// which for ethernet is 14 and not 0 like it is for rawip.
//
// For similar reasons this will fail with non-offloaded VLAN tags on < 4.19 kernels,
// since those extend the ethernet header from 14 to 18 bytes.
return KVER_IS_AT_LEAST(kver, 4, 19, 0)
? bpf_skb_load_bytes_relative(skb, L3_off, to, len, BPF_HDR_START_NET)
: bpf_skb_load_bytes(skb, L3_off, to, len);
}
static __always_inline inline void do_packet_tracing(
const struct __sk_buff* const skb, const struct egress_bool egress, const uint32_t uid,
const uint32_t tag, const bool enable_tracing, const struct kver_uint kver) {
if (!enable_tracing) return;
if (!KVER_IS_AT_LEAST(kver, 5, 8, 0)) return;
uint32_t mapKey = 0;
bool* traceConfig = bpf_packet_trace_enabled_map_lookup_elem(&mapKey);
if (traceConfig == NULL) return;
if (*traceConfig == false) return;
PacketTrace* pkt = bpf_packet_trace_ringbuf_reserve();
if (pkt == NULL) return;
// Errors from bpf_skb_load_bytes_net are ignored to favor returning something
// over returning nothing. In the event of an error, the kernel will fill in
// zero for the destination memory. Do not change the default '= 0' below.
uint8_t proto = 0;
uint8_t L4_off = 0;
uint8_t ipVersion = 0;
if (skb->protocol == htons(ETH_P_IP)) {
(void)bpf_skb_load_bytes_net(skb, IP4_OFFSET(protocol), &proto, sizeof(proto), kver);
(void)bpf_skb_load_bytes_net(skb, IPPROTO_IHL_OFF, &L4_off, sizeof(L4_off), kver);
L4_off = (L4_off & 0x0F) * 4; // IHL calculation.
ipVersion = 4;
} else if (skb->protocol == htons(ETH_P_IPV6)) {
(void)bpf_skb_load_bytes_net(skb, IP6_OFFSET(nexthdr), &proto, sizeof(proto), kver);
L4_off = sizeof(struct ipv6hdr);
ipVersion = 6;
// skip over a *single* HOPOPTS or DSTOPTS extension header (if present)
if (proto == IPPROTO_HOPOPTS || proto == IPPROTO_DSTOPTS) {
struct {
uint8_t proto, len;
} ext_hdr;
if (!bpf_skb_load_bytes_net(skb, L4_off, &ext_hdr, sizeof(ext_hdr), kver)) {
proto = ext_hdr.proto;
L4_off += (ext_hdr.len + 1) * 8;
}
}
}
uint8_t flags = 0;
__be16 sport = 0, dport = 0;
if (L4_off >= 20) {
switch (proto) {
case IPPROTO_TCP:
(void)bpf_skb_load_bytes_net(skb, L4_off + TCP_FLAG8_OFF, &flags, sizeof(flags), kver);
// fallthrough
case IPPROTO_DCCP:
case IPPROTO_UDP:
case IPPROTO_UDPLITE:
case IPPROTO_SCTP:
// all of these L4 protocols start with be16 src & dst port
(void)bpf_skb_load_bytes_net(skb, L4_off + 0, &sport, sizeof(sport), kver);
(void)bpf_skb_load_bytes_net(skb, L4_off + 2, &dport, sizeof(dport), kver);
break;
case IPPROTO_ICMP:
case IPPROTO_ICMPV6:
// Both IPv4 and IPv6 icmp start with u8 type & code, which we store in the bottom
// (ie. second) byte of sport/dport (which are be16s), the top byte is already zero.
(void)bpf_skb_load_bytes_net(skb, L4_off + 0, (char *)&sport + 1, 1, kver); //type
(void)bpf_skb_load_bytes_net(skb, L4_off + 1, (char *)&dport + 1, 1, kver); //code
break;
}
}
pkt->timestampNs = bpf_ktime_get_boot_ns();
pkt->ifindex = skb->ifindex;
pkt->length = skb->len;
pkt->uid = uid;
pkt->tag = tag;
pkt->sport = sport;
pkt->dport = dport;
pkt->egress = egress.egress;
pkt->wakeup = !egress.egress && (skb->mark & 0x80000000); // Fwmark.ingress_cpu_wakeup
pkt->ipProto = proto;
pkt->tcpFlags = flags;
pkt->ipVersion = ipVersion;
bpf_packet_trace_ringbuf_submit(pkt);
}
static __always_inline inline bool skip_owner_match(struct __sk_buff* skb,
const struct egress_bool egress,
const struct kver_uint kver) {
uint32_t flag = 0;
if (skb->protocol == htons(ETH_P_IP)) {
uint8_t proto;
// no need to check for success, proto will be zeroed if bpf_skb_load_bytes_net() fails
(void)bpf_skb_load_bytes_net(skb, IP4_OFFSET(protocol), &proto, sizeof(proto), kver);
if (proto == IPPROTO_ESP) return true;
if (proto != IPPROTO_TCP) return false; // handles read failure above
uint8_t ihl;
// we don't check for success, as this cannot fail, as it is earlier in the packet than
// proto, the reading of which must have succeeded, additionally the next read
// (a little bit deeper in the packet in spite of ihl being zeroed) of the tcp flags
// field will also fail, and that failure we already handle correctly
// (we also don't check that ihl in [0x45,0x4F] nor that ipv4 header checksum is correct)
(void)bpf_skb_load_bytes_net(skb, IPPROTO_IHL_OFF, &ihl, sizeof(ihl), kver);
// if the read below fails, we'll just assume no TCP flags are set, which is fine.
(void)bpf_skb_load_bytes_net(skb, (ihl & 0xF) * 4 + TCP_FLAG32_OFF,
&flag, sizeof(flag), kver);
} else if (skb->protocol == htons(ETH_P_IPV6)) {
uint8_t proto;
// no need to check for success, proto will be zeroed if bpf_skb_load_bytes_net() fails
(void)bpf_skb_load_bytes_net(skb, IP6_OFFSET(nexthdr), &proto, sizeof(proto), kver);
if (proto == IPPROTO_ESP) return true;
if (proto != IPPROTO_TCP) return false; // handles read failure above
// if the read below fails, we'll just assume no TCP flags are set, which is fine.
(void)bpf_skb_load_bytes_net(skb, sizeof(struct ipv6hdr) + TCP_FLAG32_OFF,
&flag, sizeof(flag), kver);
} else {
return false;
}
// Always allow RST's, and additionally allow ingress FINs
return flag & (TCP_FLAG_RST | (egress.egress ? 0 : TCP_FLAG_FIN)); // false on read failure
}
static __always_inline inline BpfConfig getConfig(uint32_t configKey) {
uint32_t mapSettingKey = configKey;
BpfConfig* config = bpf_configuration_map_lookup_elem(&mapSettingKey);
if (!config) {
// Couldn't read configuration entry. Assume everything is disabled.
return DEFAULT_CONFIG;
}
return *config;
}
static __always_inline inline bool ingress_should_discard(struct __sk_buff* skb,
const struct kver_uint kver) {
// Require 4.19, since earlier kernels don't have bpf_skb_load_bytes_relative() which
// provides relative to L3 header reads. Without that we could fetch the wrong bytes.
// Additionally earlier bpf verifiers are much harder to please.
if (!KVER_IS_AT_LEAST(kver, 4, 19, 0)) return false;
IngressDiscardKey k = {};
if (skb->protocol == htons(ETH_P_IP)) {
k.daddr.s6_addr32[2] = htonl(0xFFFF);
(void)bpf_skb_load_bytes_net(skb, IP4_OFFSET(daddr), &k.daddr.s6_addr32[3], 4, kver);
} else if (skb->protocol == htons(ETH_P_IPV6)) {
(void)bpf_skb_load_bytes_net(skb, IP6_OFFSET(daddr), &k.daddr, sizeof(k.daddr), kver);
} else {
return false; // non IPv4/IPv6, so no IP to match on
}
// we didn't check for load success, because destination bytes will be zeroed if
// bpf_skb_load_bytes_net() fails, instead we rely on daddr of '::' and '::ffff:0.0.0.0'
// never being present in the map itself
IngressDiscardValue* v = bpf_ingress_discard_map_lookup_elem(&k);
if (!v) return false; // lookup failure -> no protection in place -> allow
// if (skb->ifindex == 1) return false; // allow 'lo', but can't happen - see callsite
if (skb->ifindex == v->iif[0]) return false; // allowed interface
if (skb->ifindex == v->iif[1]) return false; // allowed interface
return true; // disallowed interface
}
static __always_inline inline int bpf_owner_match(struct __sk_buff* skb, uint32_t uid,
const struct egress_bool egress,
const struct kver_uint kver) {
if (is_system_uid(uid)) return PASS;
if (skip_owner_match(skb, egress, kver)) return PASS;
BpfConfig enabledRules = getConfig(UID_RULES_CONFIGURATION_KEY);
// BACKGROUND match does not apply to loopback traffic
if (skb->ifindex == 1) enabledRules &= ~BACKGROUND_MATCH;
UidOwnerValue* uidEntry = bpf_uid_owner_map_lookup_elem(&uid);
uint32_t uidRules = uidEntry ? uidEntry->rule : 0;
uint32_t allowed_iif = uidEntry ? uidEntry->iif : 0;
if (isBlockedByUidRules(enabledRules, uidRules)) return DROP;
if (!egress.egress && skb->ifindex != 1) {
if (ingress_should_discard(skb, kver)) return DROP;
if (uidRules & IIF_MATCH) {
if (allowed_iif && skb->ifindex != allowed_iif) {
// Drops packets not coming from lo nor the allowed interface
// allowed interface=0 is a wildcard and does not drop packets
return DROP_UNLESS_DNS;
}
} else if (uidRules & LOCKDOWN_VPN_MATCH) {
// Drops packets not coming from lo and rule does not have IIF_MATCH but has
// LOCKDOWN_VPN_MATCH
return DROP_UNLESS_DNS;
}
}
return PASS;
}
static __always_inline inline void update_stats_with_config(const uint32_t selectedMap,
const struct __sk_buff* const skb,
const StatsKey* const key,
const struct egress_bool egress,
const struct kver_uint kver) {
if (selectedMap == SELECT_MAP_A) {
update_stats_map_A(skb, key, egress, kver);
} else {
update_stats_map_B(skb, key, egress, kver);
}
}
static __always_inline inline int bpf_traffic_account(struct __sk_buff* skb,
const struct egress_bool egress,
const bool enable_tracing,
const struct kver_uint kver) {
// sock_uid will be 'overflowuid' if !sk_fullsock(sk_to_full_sk(skb->sk))
uint32_t sock_uid = bpf_get_socket_uid(skb);
// kernel's DEFAULT_OVERFLOWUID is 65534, this is the overflow 'nobody' uid,
// usually this being returned means that skb->sk is NULL during RX
// (early decap socket lookup failure), which commonly happens for incoming
// packets to an unconnected udp socket.
// But it can also happen for egress from a timewait socket.
// Let's treat such cases as 'root' which is_system_uid()
if (sock_uid == 65534) sock_uid = 0;
uint64_t cookie = bpf_get_socket_cookie(skb); // 0 iff !skb->sk
UidTagValue* utag = bpf_cookie_tag_map_lookup_elem(&cookie);
uint32_t uid, tag;
if (utag) {
uid = utag->uid;
tag = utag->tag;
} else {
uid = sock_uid;
tag = 0;
}
// Always allow and never count clat traffic. Only the IPv4 traffic on the stacked
// interface is accounted for and subject to usage restrictions.
// CLAT IPv6 TX sockets are *always* tagged with CLAT uid, see tagSocketAsClat()
// CLAT daemon receives via an untagged AF_PACKET socket.
if (egress.egress && uid == AID_CLAT) return PASS;
int match = bpf_owner_match(skb, sock_uid, egress, kver);
// Workaround for secureVPN with VpnIsolation enabled, refer to b/159994981 for details.
// Keep TAG_SYSTEM_DNS in sync with DnsResolver/include/netd_resolv/resolv.h
// and TrafficStatsConstants.java
#define TAG_SYSTEM_DNS 0xFFFFFF82
if (tag == TAG_SYSTEM_DNS && uid == AID_DNS) {
uid = sock_uid;
if (match == DROP_UNLESS_DNS) match = PASS;
} else {
if (match == DROP_UNLESS_DNS) match = DROP;
}
// If an outbound packet is going to be dropped, we do not count that traffic.
if (egress.egress && (match == DROP)) return DROP;
StatsKey key = {.uid = uid, .tag = tag, .counterSet = 0, .ifaceIndex = skb->ifindex};
uint8_t* counterSet = bpf_uid_counterset_map_lookup_elem(&uid);
if (counterSet) key.counterSet = (uint32_t)*counterSet;
uint32_t mapSettingKey = CURRENT_STATS_MAP_CONFIGURATION_KEY;
uint32_t* selectedMap = bpf_configuration_map_lookup_elem(&mapSettingKey);
if (!selectedMap) return PASS; // cannot happen, needed to keep bpf verifier happy
do_packet_tracing(skb, egress, uid, tag, enable_tracing, kver);
update_stats_with_config(*selectedMap, skb, &key, egress, kver);
update_app_uid_stats_map(skb, &uid, egress, kver);
// We've already handled DROP_UNLESS_DNS up above, thus when we reach here the only
// possible values of match are DROP(0) or PASS(1), however we need to use
// "match &= 1" before 'return match' to help the kernel's bpf verifier,
// so that it can be 100% certain that the returned value is always 0 or 1.
// We use assembly so that it cannot be optimized out by a too smart compiler.
asm("%0 &= 1" : "+r"(match));
return match;
}
// Tracing on Android U+ 5.8+
DEFINE_BPF_PROG_EXT("cgroupskb/ingress/stats$trace", AID_ROOT, AID_SYSTEM,
bpf_cgroup_ingress_trace, KVER_5_8, KVER_INF,
BPFLOADER_MAINLINE_U_VERSION, BPFLOADER_MAX_VER, MANDATORY,
"fs_bpf_netd_readonly", "",
LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, INGRESS, TRACE_ON, KVER_5_8);
}
DEFINE_NETD_BPF_PROG_KVER_RANGE("cgroupskb/ingress/stats$4_19", AID_ROOT, AID_SYSTEM,
bpf_cgroup_ingress_4_19, KVER_4_19, KVER_INF)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, INGRESS, TRACE_OFF, KVER_4_19);
}
DEFINE_NETD_BPF_PROG_KVER_RANGE("cgroupskb/ingress/stats$4_14", AID_ROOT, AID_SYSTEM,
bpf_cgroup_ingress_4_14, KVER_NONE, KVER_4_19)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, INGRESS, TRACE_OFF, KVER_NONE);
}
// Tracing on Android U+ 5.8+
DEFINE_BPF_PROG_EXT("cgroupskb/egress/stats$trace", AID_ROOT, AID_SYSTEM,
bpf_cgroup_egress_trace, KVER_5_8, KVER_INF,
BPFLOADER_MAINLINE_U_VERSION, BPFLOADER_MAX_VER, MANDATORY,
"fs_bpf_netd_readonly", "",
LOAD_ON_ENG, LOAD_ON_USER, LOAD_ON_USERDEBUG)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, EGRESS, TRACE_ON, KVER_5_8);
}
DEFINE_NETD_BPF_PROG_KVER_RANGE("cgroupskb/egress/stats$4_19", AID_ROOT, AID_SYSTEM,
bpf_cgroup_egress_4_19, KVER_4_19, KVER_INF)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, EGRESS, TRACE_OFF, KVER_4_19);
}
DEFINE_NETD_BPF_PROG_KVER_RANGE("cgroupskb/egress/stats$4_14", AID_ROOT, AID_SYSTEM,
bpf_cgroup_egress_4_14, KVER_NONE, KVER_4_19)
(struct __sk_buff* skb) {
return bpf_traffic_account(skb, EGRESS, TRACE_OFF, KVER_NONE);
}
// WARNING: Android T's non-updatable netd depends on the name of this program.
DEFINE_XTBPF_PROG("skfilter/egress/xtbpf", AID_ROOT, AID_NET_ADMIN, xt_bpf_egress_prog)
(struct __sk_buff* skb) {
// Clat daemon does not generate new traffic, all its traffic is accounted for already
// on the v4-* interfaces (except for the 20 (or 28) extra bytes of IPv6 vs IPv4 overhead,
// but that can be corrected for later when merging v4-foo stats into interface foo's).
// CLAT sockets are created by system server and tagged as uid CLAT, see tagSocketAsClat()
uint32_t sock_uid = bpf_get_socket_uid(skb);
if (sock_uid == AID_SYSTEM) {
uint64_t cookie = bpf_get_socket_cookie(skb);
UidTagValue* utag = bpf_cookie_tag_map_lookup_elem(&cookie);
if (utag && utag->uid == AID_CLAT) return XTBPF_NOMATCH;
}
uint32_t key = skb->ifindex;
update_iface_stats_map(skb, &key, EGRESS, KVER_NONE);
return XTBPF_MATCH;
}
// WARNING: Android T's non-updatable netd depends on the name of this program.
DEFINE_XTBPF_PROG("skfilter/ingress/xtbpf", AID_ROOT, AID_NET_ADMIN, xt_bpf_ingress_prog)
(struct __sk_buff* skb) {
// Clat daemon traffic is not accounted by virtue of iptables raw prerouting drop rule
// (in clat_raw_PREROUTING chain), which triggers before this (in bw_raw_PREROUTING chain).
// It will be accounted for on the v4-* clat interface instead.
// Keep that in mind when moving this out of iptables xt_bpf and into tc ingress (or xdp).
uint32_t key = skb->ifindex;
update_iface_stats_map(skb, &key, INGRESS, KVER_NONE);
return XTBPF_MATCH;
}
DEFINE_SYS_BPF_PROG("schedact/ingress/account", AID_ROOT, AID_NET_ADMIN,
tc_bpf_ingress_account_prog)
(struct __sk_buff* skb) {
if (is_received_skb(skb)) {
// Account for ingress traffic before tc drops it.
uint32_t key = skb->ifindex;
update_iface_stats_map(skb, &key, INGRESS, KVER_NONE);
}
return TC_ACT_UNSPEC;
}
// WARNING: Android T's non-updatable netd depends on the name of this program.
DEFINE_XTBPF_PROG("skfilter/allowlist/xtbpf", AID_ROOT, AID_NET_ADMIN, xt_bpf_allowlist_prog)
(struct __sk_buff* skb) {
uint32_t sock_uid = bpf_get_socket_uid(skb);
if (is_system_uid(sock_uid)) return XTBPF_MATCH;
// kernel's DEFAULT_OVERFLOWUID is 65534, this is the overflow 'nobody' uid,
// usually this being returned means that skb->sk is NULL during RX
// (early decap socket lookup failure), which commonly happens for incoming
// packets to an unconnected udp socket.
// But it can also happen for egress from a timewait socket.
// Let's treat such cases as 'root' which is_system_uid()
if (sock_uid == 65534) return XTBPF_MATCH;
UidOwnerValue* allowlistMatch = bpf_uid_owner_map_lookup_elem(&sock_uid);
if (allowlistMatch) return allowlistMatch->rule & HAPPY_BOX_MATCH ? XTBPF_MATCH : XTBPF_NOMATCH;
return XTBPF_NOMATCH;
}
// WARNING: Android T's non-updatable netd depends on the name of this program.
DEFINE_XTBPF_PROG("skfilter/denylist/xtbpf", AID_ROOT, AID_NET_ADMIN, xt_bpf_denylist_prog)
(struct __sk_buff* skb) {
uint32_t sock_uid = bpf_get_socket_uid(skb);
UidOwnerValue* denylistMatch = bpf_uid_owner_map_lookup_elem(&sock_uid);
uint32_t penalty_box = PENALTY_BOX_USER_MATCH | PENALTY_BOX_ADMIN_MATCH;
if (denylistMatch) return denylistMatch->rule & penalty_box ? XTBPF_MATCH : XTBPF_NOMATCH;
return XTBPF_NOMATCH;
}
static __always_inline inline uint8_t get_app_permissions() {
uint64_t gid_uid = bpf_get_current_uid_gid();
/*
* A given app is guaranteed to have the same app ID in all the profiles in
* which it is installed, and install permission is granted to app for all
* user at install time so we only check the appId part of a request uid at
* run time. See UserHandle#isSameApp for detail.
*/
uint32_t appId = (gid_uid & 0xffffffff) % AID_USER_OFFSET; // == PER_USER_RANGE == 100000
uint8_t* permissions = bpf_uid_permission_map_lookup_elem(&appId);
// if UID not in map, then default to just INTERNET permission.
return permissions ? *permissions : BPF_PERMISSION_INTERNET;
}
DEFINE_NETD_BPF_PROG_KVER("cgroupsock/inet_create", AID_ROOT, AID_ROOT, inet_socket_create,
KVER_4_14)
(__unused struct bpf_sock* sk) {
return (get_app_permissions() & BPF_PERMISSION_INTERNET) ? BPF_ALLOW : BPF_DISALLOW;
}
DEFINE_NETD_V_BPF_PROG_KVER("cgroupsockrelease/inet_release", AID_ROOT, AID_ROOT,
inet_socket_release, KVER_5_10)
(struct bpf_sock* sk) {
uint64_t cookie = bpf_get_sk_cookie(sk);
if (cookie) bpf_cookie_tag_map_delete_elem(&cookie);
return 1;
}
static __always_inline inline int check_localhost(__unused struct bpf_sock_addr *ctx) {
// See include/uapi/linux/bpf.h:
//
// struct bpf_sock_addr {
// __u32 user_family; // R: 4 byte
// __u32 user_ip4; // BE, R: 1,2,4-byte, W: 4-byte
// __u32 user_ip6[4]; // BE, R: 1,2,4,8-byte, W: 4,8-byte
// __u32 user_port; // BE, R: 1,2,4-byte, W: 4-byte
// __u32 family; // R: 4 byte
// __u32 type; // R: 4 byte
// __u32 protocol; // R: 4 byte
// __u32 msg_src_ip4; // BE, R: 1,2,4-byte, W: 4-byte
// __u32 msg_src_ip6[4]; // BE, R: 1,2,4,8-byte, W: 4,8-byte
// __bpf_md_ptr(struct bpf_sock *, sk);
// };
return BPF_ALLOW;
}
DEFINE_NETD_V_BPF_PROG_KVER("connect4/inet4_connect", AID_ROOT, AID_ROOT, inet4_connect, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("connect6/inet6_connect", AID_ROOT, AID_ROOT, inet6_connect, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("recvmsg4/udp4_recvmsg", AID_ROOT, AID_ROOT, udp4_recvmsg, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("recvmsg6/udp6_recvmsg", AID_ROOT, AID_ROOT, udp6_recvmsg, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("sendmsg4/udp4_sendmsg", AID_ROOT, AID_ROOT, udp4_sendmsg, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("sendmsg6/udp6_sendmsg", AID_ROOT, AID_ROOT, udp6_sendmsg, KVER_4_14)
(struct bpf_sock_addr *ctx) {
return check_localhost(ctx);
}
DEFINE_NETD_V_BPF_PROG_KVER("getsockopt/prog", AID_ROOT, AID_ROOT, getsockopt_prog, KVER_5_4)
(struct bpf_sockopt *ctx) {
// Tell kernel to return 'original' kernel reply (instead of the bpf modified buffer)
// This is important if the answer is larger than PAGE_SIZE (max size this bpf hook can provide)
ctx->optlen = 0;
return BPF_ALLOW;
}
DEFINE_NETD_V_BPF_PROG_KVER("setsockopt/prog", AID_ROOT, AID_ROOT, setsockopt_prog, KVER_5_4)
(struct bpf_sockopt *ctx) {
// Tell kernel to use/process original buffer provided by userspace.
// This is important if it is larger than PAGE_SIZE (max size this bpf hook can handle).
ctx->optlen = 0;
return BPF_ALLOW;
}
LICENSE("Apache 2.0");
CRITICAL("Connectivity and netd");

274
bpf/progs/netd.h Normal file
View file

@ -0,0 +1,274 @@
/*
* Copyright (C) 2018 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <cutils/android_filesystem_config.h>
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/in.h>
#include <linux/in6.h>
#ifdef __cplusplus
#include <string_view>
#include "XtBpfProgLocations.h"
#endif
// This header file is shared by eBPF kernel programs (C) and netd (C++) and
// some of the maps are also accessed directly from Java mainline module code.
//
// Hence: explicitly pad all relevant structures and assert that their size
// is the sum of the sizes of their fields.
#define STRUCT_SIZE(name, size) _Static_assert(sizeof(name) == (size), "Incorrect struct size.")
typedef struct {
uint32_t uid;
uint32_t tag;
} UidTagValue;
STRUCT_SIZE(UidTagValue, 2 * 4); // 8
typedef struct {
uint32_t uid;
uint32_t tag;
uint32_t counterSet;
uint32_t ifaceIndex;
} StatsKey;
STRUCT_SIZE(StatsKey, 4 * 4); // 16
typedef struct {
uint64_t rxPackets;
uint64_t rxBytes;
uint64_t txPackets;
uint64_t txBytes;
} StatsValue;
STRUCT_SIZE(StatsValue, 4 * 8); // 32
#ifdef __cplusplus
static inline StatsValue& operator+=(StatsValue& lhs, const StatsValue& rhs) {
lhs.rxPackets += rhs.rxPackets;
lhs.rxBytes += rhs.rxBytes;
lhs.txPackets += rhs.txPackets;
lhs.txBytes += rhs.txBytes;
return lhs;
}
#endif
typedef struct {
char name[IFNAMSIZ];
} IfaceValue;
STRUCT_SIZE(IfaceValue, 16);
typedef struct {
uint64_t timestampNs;
uint32_t ifindex;
uint32_t length;
uint32_t uid;
uint32_t tag;
__be16 sport;
__be16 dport;
bool egress:1,
wakeup:1;
uint8_t ipProto;
uint8_t tcpFlags;
uint8_t ipVersion; // 4=IPv4, 6=IPv6, 0=unknown
} PacketTrace;
STRUCT_SIZE(PacketTrace, 8+4+4 + 4+4 + 2+2 + 1+1+1+1);
// Since we cannot garbage collect the stats map since device boot, we need to make these maps as
// large as possible. The maximum size of number of map entries we can have is depend on the rlimit
// of MEM_LOCK granted to netd. The memory space needed by each map can be calculated by the
// following fomula:
// elem_size = 40 + roundup(key_size, 8) + roundup(value_size, 8)
// cost = roundup_pow_of_two(max_entries) * 16 + elem_size * max_entries +
// elem_size * number_of_CPU
// And the cost of each map currently used is(assume the device have 8 CPUs):
// cookie_tag_map: key: 8 bytes, value: 8 bytes, cost: 822592 bytes = 823Kbytes
// uid_counter_set_map: key: 4 bytes, value: 1 bytes, cost: 145216 bytes = 145Kbytes
// app_uid_stats_map: key: 4 bytes, value: 32 bytes, cost: 1062784 bytes = 1063Kbytes
// uid_stats_map: key: 16 bytes, value: 32 bytes, cost: 1142848 bytes = 1143Kbytes
// tag_stats_map: key: 16 bytes, value: 32 bytes, cost: 1142848 bytes = 1143Kbytes
// iface_index_name_map:key: 4 bytes, value: 16 bytes, cost: 80896 bytes = 81Kbytes
// iface_stats_map: key: 4 bytes, value: 32 bytes, cost: 97024 bytes = 97Kbytes
// dozable_uid_map: key: 4 bytes, value: 1 bytes, cost: 145216 bytes = 145Kbytes
// standby_uid_map: key: 4 bytes, value: 1 bytes, cost: 145216 bytes = 145Kbytes
// powersave_uid_map: key: 4 bytes, value: 1 bytes, cost: 145216 bytes = 145Kbytes
// packet_trace_ringbuf:key: 0 bytes, value: 24 bytes, cost: 32768 bytes = 32Kbytes
// total: 4962Kbytes
// It takes maximum 4.9MB kernel memory space if all maps are full, which requires any devices
// running this module to have a memlock rlimit to be larger then 5MB. In the old qtaguid module,
// we don't have a total limit for data entries but only have limitation of tags each uid can have.
// (default is 1024 in kernel);
// 'static' - otherwise these constants end up in .rodata in the resulting .o post compilation
static const int COOKIE_UID_MAP_SIZE = 10000;
static const int UID_COUNTERSET_MAP_SIZE = 4000;
static const int APP_STATS_MAP_SIZE = 10000;
static const int STATS_MAP_SIZE = 5000;
static const int IFACE_INDEX_NAME_MAP_SIZE = 1000;
static const int IFACE_STATS_MAP_SIZE = 1000;
static const int CONFIGURATION_MAP_SIZE = 2;
static const int UID_OWNER_MAP_SIZE = 4000;
static const int INGRESS_DISCARD_MAP_SIZE = 100;
static const int PACKET_TRACE_BUF_SIZE = 32 * 1024;
static const int DATA_SAVER_ENABLED_MAP_SIZE = 1;
#ifdef __cplusplus
#define BPF_NETD_PATH "/sys/fs/bpf/netd_shared/"
#define BPF_EGRESS_PROG_PATH BPF_NETD_PATH "prog_netd_cgroupskb_egress_stats"
#define BPF_INGRESS_PROG_PATH BPF_NETD_PATH "prog_netd_cgroupskb_ingress_stats"
#define ASSERT_STRING_EQUAL(s1, s2) \
static_assert(std::string_view(s1) == std::string_view(s2), "mismatch vs Android T netd")
/* -=-=-=-=- WARNING -=-=-=-=-
*
* These 4 xt_bpf program paths are actually defined by:
* //system/netd/include/mainline/XtBpfProgLocations.h
* which is intentionally a non-automerged location.
*
* They are *UNCHANGEABLE* due to being hard coded in Android T's netd binary
* as such we have compile time asserts that things match.
* (which will be validated during build on mainline-prod branch against old system/netd)
*
* If you break this, netd on T will fail to start with your tethering mainline module.
*/
ASSERT_STRING_EQUAL(XT_BPF_INGRESS_PROG_PATH, BPF_NETD_PATH "prog_netd_skfilter_ingress_xtbpf");
ASSERT_STRING_EQUAL(XT_BPF_EGRESS_PROG_PATH, BPF_NETD_PATH "prog_netd_skfilter_egress_xtbpf");
ASSERT_STRING_EQUAL(XT_BPF_ALLOWLIST_PROG_PATH, BPF_NETD_PATH "prog_netd_skfilter_allowlist_xtbpf");
ASSERT_STRING_EQUAL(XT_BPF_DENYLIST_PROG_PATH, BPF_NETD_PATH "prog_netd_skfilter_denylist_xtbpf");
#define CGROUP_INET_CREATE_PROG_PATH BPF_NETD_PATH "prog_netd_cgroupsock_inet_create"
#define CGROUP_INET_RELEASE_PROG_PATH BPF_NETD_PATH "prog_netd_cgroupsockrelease_inet_release"
#define CGROUP_CONNECT4_PROG_PATH BPF_NETD_PATH "prog_netd_connect4_inet4_connect"
#define CGROUP_CONNECT6_PROG_PATH BPF_NETD_PATH "prog_netd_connect6_inet6_connect"
#define CGROUP_UDP4_RECVMSG_PROG_PATH BPF_NETD_PATH "prog_netd_recvmsg4_udp4_recvmsg"
#define CGROUP_UDP6_RECVMSG_PROG_PATH BPF_NETD_PATH "prog_netd_recvmsg6_udp6_recvmsg"
#define CGROUP_UDP4_SENDMSG_PROG_PATH BPF_NETD_PATH "prog_netd_sendmsg4_udp4_sendmsg"
#define CGROUP_UDP6_SENDMSG_PROG_PATH BPF_NETD_PATH "prog_netd_sendmsg6_udp6_sendmsg"
#define CGROUP_GETSOCKOPT_PROG_PATH BPF_NETD_PATH "prog_netd_getsockopt_prog"
#define CGROUP_SETSOCKOPT_PROG_PATH BPF_NETD_PATH "prog_netd_setsockopt_prog"
#define TC_BPF_INGRESS_ACCOUNT_PROG_NAME "prog_netd_schedact_ingress_account"
#define TC_BPF_INGRESS_ACCOUNT_PROG_PATH BPF_NETD_PATH TC_BPF_INGRESS_ACCOUNT_PROG_NAME
#define COOKIE_TAG_MAP_PATH BPF_NETD_PATH "map_netd_cookie_tag_map"
#define UID_COUNTERSET_MAP_PATH BPF_NETD_PATH "map_netd_uid_counterset_map"
#define APP_UID_STATS_MAP_PATH BPF_NETD_PATH "map_netd_app_uid_stats_map"
#define STATS_MAP_A_PATH BPF_NETD_PATH "map_netd_stats_map_A"
#define STATS_MAP_B_PATH BPF_NETD_PATH "map_netd_stats_map_B"
#define IFACE_INDEX_NAME_MAP_PATH BPF_NETD_PATH "map_netd_iface_index_name_map"
#define IFACE_STATS_MAP_PATH BPF_NETD_PATH "map_netd_iface_stats_map"
#define CONFIGURATION_MAP_PATH BPF_NETD_PATH "map_netd_configuration_map"
#define UID_OWNER_MAP_PATH BPF_NETD_PATH "map_netd_uid_owner_map"
#define UID_PERMISSION_MAP_PATH BPF_NETD_PATH "map_netd_uid_permission_map"
#define INGRESS_DISCARD_MAP_PATH BPF_NETD_PATH "map_netd_ingress_discard_map"
#define PACKET_TRACE_RINGBUF_PATH BPF_NETD_PATH "map_netd_packet_trace_ringbuf"
#define PACKET_TRACE_ENABLED_MAP_PATH BPF_NETD_PATH "map_netd_packet_trace_enabled_map"
#define DATA_SAVER_ENABLED_MAP_PATH BPF_NETD_PATH "map_netd_data_saver_enabled_map"
#endif // __cplusplus
// LINT.IfChange(match_type)
enum UidOwnerMatchType : uint32_t {
NO_MATCH = 0,
HAPPY_BOX_MATCH = (1 << 0),
PENALTY_BOX_USER_MATCH = (1 << 1),
DOZABLE_MATCH = (1 << 2),
STANDBY_MATCH = (1 << 3),
POWERSAVE_MATCH = (1 << 4),
RESTRICTED_MATCH = (1 << 5),
LOW_POWER_STANDBY_MATCH = (1 << 6),
IIF_MATCH = (1 << 7),
LOCKDOWN_VPN_MATCH = (1 << 8),
OEM_DENY_1_MATCH = (1 << 9),
OEM_DENY_2_MATCH = (1 << 10),
OEM_DENY_3_MATCH = (1 << 11),
BACKGROUND_MATCH = (1 << 12),
PENALTY_BOX_ADMIN_MATCH = (1 << 13),
};
// LINT.ThenChange(../framework/src/android/net/BpfNetMapsConstants.java)
enum BpfPermissionMatch : uint8_t {
BPF_PERMISSION_INTERNET = 1 << 2,
BPF_PERMISSION_UPDATE_DEVICE_STATS = 1 << 3,
};
// In production we use two identical stats maps to record per uid stats and
// do swap and clean based on the configuration specified here. The statsMapType
// value in configuration map specified which map is currently in use.
enum StatsMapType : uint32_t {
SELECT_MAP_A,
SELECT_MAP_B,
};
// TODO: change the configuration object from a bitmask to an object with clearer
// semantics, like a struct.
typedef uint32_t BpfConfig;
static const BpfConfig DEFAULT_CONFIG = 0;
typedef struct {
// Allowed interface index. Only applicable if IIF_MATCH is set in the rule bitmask above.
uint32_t iif;
// A bitmask of enum values in UidOwnerMatchType.
uint32_t rule;
} UidOwnerValue;
STRUCT_SIZE(UidOwnerValue, 2 * 4); // 8
typedef struct {
// The destination ip of the incoming packet. IPv4 uses IPv4-mapped IPv6 address format.
struct in6_addr daddr;
} IngressDiscardKey;
STRUCT_SIZE(IngressDiscardKey, 16); // 16
typedef struct {
// Allowed interface indexes. Use same value multiple times if you just want to match 1 value.
uint32_t iif[2];
} IngressDiscardValue;
STRUCT_SIZE(IngressDiscardValue, 2 * 4); // 8
// Entry in the configuration map that stores which UID rules are enabled.
#define UID_RULES_CONFIGURATION_KEY 0
// Entry in the configuration map that stores which stats map is currently in use.
#define CURRENT_STATS_MAP_CONFIGURATION_KEY 1
// Entry in the data saver enabled map that stores whether data saver is enabled or not.
#define DATA_SAVER_ENABLED_KEY 0
#undef STRUCT_SIZE
// DROP_IF_SET is set of rules that DROP if rule is globally enabled, and per-uid bit is set
#define DROP_IF_SET (STANDBY_MATCH | OEM_DENY_1_MATCH | OEM_DENY_2_MATCH | OEM_DENY_3_MATCH)
// DROP_IF_UNSET is set of rules that should DROP if globally enabled, and per-uid bit is NOT set
#define DROP_IF_UNSET (DOZABLE_MATCH | POWERSAVE_MATCH | RESTRICTED_MATCH \
| LOW_POWER_STANDBY_MATCH | BACKGROUND_MATCH)
// Warning: funky bit-wise arithmetic: in parallel, for all DROP_IF_SET/UNSET rules
// check whether the rules are globally enabled, and if so whether the rules are
// set/unset for the specific uid. DROP if that is the case for ANY of the rules.
// We achieve this by masking out only the bits/rules we're interested in checking,
// and negating (via bit-wise xor) the bits/rules that should drop if unset.
static inline bool isBlockedByUidRules(BpfConfig enabledRules, uint32_t uidRules) {
return enabledRules & (DROP_IF_SET | DROP_IF_UNSET) & (uidRules ^ DROP_IF_UNSET);
}
static inline bool is_system_uid(uint32_t uid) {
// MIN_SYSTEM_UID is AID_ROOT == 0, so uint32_t is *always* >= 0
// MAX_SYSTEM_UID is AID_NOBODY == 9999, while AID_APP_START == 10000
return ((uid % AID_USER_OFFSET) < AID_APP_START);
}

824
bpf/progs/offload.c Normal file
View file

@ -0,0 +1,824 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef MAINLINE
// BTF is incompatible with bpfloaders < v0.10, hence for S (v0.2) we must
// ship a different file than for later versions, but we need bpfloader v0.25+
// for obj@ver.o support
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#else /* MAINLINE */
// The resulting .o needs to load on the Android S bpfloader
#define BPFLOADER_MIN_VER BPFLOADER_S_VERSION
#define BPFLOADER_MAX_VER BPFLOADER_T_VERSION
#endif /* MAINLINE */
#include "bpf_net_helpers.h"
#include "offload.h"
// ----- Tethering Error Counters -----
// Note that pre-T devices with Mediatek chipsets may have a kernel bug (bad patch
// "[ALPS05162612] bpf: fix ubsan error") making it impossible to write to non-zero
// offset of bpf map ARRAYs. This file (offload.o) loads on S+, but luckily this
// array is only written by bpf code, and only read by userspace.
DEFINE_BPF_MAP_RO(tether_error_map, ARRAY, uint32_t, uint32_t, BPF_TETHER_ERR__MAX, AID_NETWORK_STACK)
#define COUNT_AND_RETURN(counter, ret) do { \
uint32_t code = BPF_TETHER_ERR_ ## counter; \
uint32_t *count = bpf_tether_error_map_lookup_elem(&code); \
if (count) __sync_fetch_and_add(count, 1); \
return ret; \
} while(0)
#define TC_DROP(counter) COUNT_AND_RETURN(counter, TC_ACT_SHOT)
#define TC_PUNT(counter) COUNT_AND_RETURN(counter, TC_ACT_PIPE)
#define XDP_DROP(counter) COUNT_AND_RETURN(counter, XDP_DROP)
#define XDP_PUNT(counter) COUNT_AND_RETURN(counter, XDP_PASS)
// ----- Tethering Data Stats and Limits -----
// Tethering stats, indexed by upstream interface.
DEFINE_BPF_MAP_GRW(tether_stats_map, HASH, TetherStatsKey, TetherStatsValue, 16, AID_NETWORK_STACK)
// Tethering data limit, indexed by upstream interface.
// (tethering allowed when stats[iif].rxBytes + stats[iif].txBytes < limit[iif])
DEFINE_BPF_MAP_GRW(tether_limit_map, HASH, TetherLimitKey, TetherLimitValue, 16, AID_NETWORK_STACK)
// ----- IPv6 Support -----
DEFINE_BPF_MAP_GRW(tether_downstream6_map, HASH, TetherDownstream6Key, Tether6Value, 64,
AID_NETWORK_STACK)
DEFINE_BPF_MAP_GRW(tether_downstream64_map, HASH, TetherDownstream64Key, TetherDownstream64Value,
1024, AID_NETWORK_STACK)
DEFINE_BPF_MAP_GRW(tether_upstream6_map, HASH, TetherUpstream6Key, Tether6Value, 64,
AID_NETWORK_STACK)
static inline __always_inline int do_forward6(struct __sk_buff* skb,
const struct rawip_bool rawip,
const struct stream_bool stream,
__unused const struct kver_uint kver) {
const bool is_ethernet = !rawip.rawip;
// Must be meta-ethernet IPv6 frame
if (skb->protocol != htons(ETH_P_IPV6)) return TC_ACT_PIPE;
// Require ethernet dst mac address to be our unicast address.
if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;
const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
// Since the program never writes via DPA (direct packet access) auto-pull/unclone logic does
// not trigger and thus we need to manually make sure we can read packet headers via DPA.
// Note: this is a blind best effort pull, which may fail or pull less - this doesn't matter.
// It has to be done early cause it will invalidate any skb->data/data_end derived pointers.
try_make_writable(skb, l2_header_size + IP6_HLEN + TCP_HLEN);
void* data = (void*)(long)skb->data;
const void* data_end = (void*)(long)skb->data_end;
struct ethhdr* eth = is_ethernet ? data : NULL; // used iff is_ethernet
struct ipv6hdr* ip6 = is_ethernet ? (void*)(eth + 1) : data;
// Must have (ethernet and) ipv6 header
if (data + l2_header_size + sizeof(*ip6) > data_end) return TC_ACT_PIPE;
// Ethertype - if present - must be IPv6
if (is_ethernet && (eth->h_proto != htons(ETH_P_IPV6))) return TC_ACT_PIPE;
// IP version must be 6
if (ip6->version != 6) TC_PUNT(INVALID_IPV6_VERSION);
// Cannot decrement during forward if already zero or would be zero,
// Let the kernel's stack handle these cases and generate appropriate ICMP errors.
if (ip6->hop_limit <= 1) TC_PUNT(LOW_TTL);
// If hardware offload is running and programming flows based on conntrack entries,
// try not to interfere with it.
if (ip6->nexthdr == IPPROTO_TCP) {
struct tcphdr* tcph = (void*)(ip6 + 1);
// Make sure we can get at the tcp header
if (data + l2_header_size + sizeof(*ip6) + sizeof(*tcph) > data_end)
TC_PUNT(INVALID_TCP_HEADER);
// Do not offload TCP packets with any one of the SYN/FIN/RST flags
if (tcph->syn || tcph->fin || tcph->rst) TC_PUNT(TCPV6_CONTROL_PACKET);
}
// Protect against forwarding packets sourced from ::1 or fe80::/64 or other weirdness.
__be32 src32 = ip6->saddr.s6_addr32[0];
if (src32 != htonl(0x0064ff9b) && // 64:ff9b:/32 incl. XLAT464 WKP
(src32 & htonl(0xe0000000)) != htonl(0x20000000)) // 2000::/3 Global Unicast
TC_PUNT(NON_GLOBAL_SRC);
// Protect against forwarding packets destined to ::1 or fe80::/64 or other weirdness.
__be32 dst32 = ip6->daddr.s6_addr32[0];
if (dst32 != htonl(0x0064ff9b) && // 64:ff9b:/32 incl. XLAT464 WKP
(dst32 & htonl(0xe0000000)) != htonl(0x20000000)) // 2000::/3 Global Unicast
TC_PUNT(NON_GLOBAL_DST);
// In the upstream direction do not forward traffic within the same /64 subnet.
if (!stream.down && (src32 == dst32) && (ip6->saddr.s6_addr32[1] == ip6->daddr.s6_addr32[1]))
TC_PUNT(LOCAL_SRC_DST);
TetherDownstream6Key kd = {
.iif = skb->ifindex,
.neigh6 = ip6->daddr,
};
TetherUpstream6Key ku = {
.iif = skb->ifindex,
// Retrieve the first 64 bits of the source IPv6 address in network order
.src64 = *(uint64_t*)&(ip6->saddr.s6_addr32[0]),
};
if (is_ethernet) __builtin_memcpy(stream.down ? kd.dstMac : ku.dstMac, eth->h_dest, ETH_ALEN);
Tether6Value* v = stream.down ? bpf_tether_downstream6_map_lookup_elem(&kd)
: bpf_tether_upstream6_map_lookup_elem(&ku);
// If we don't find any offload information then simply let the core stack handle it...
if (!v) return TC_ACT_PIPE;
uint32_t stat_and_limit_k = stream.down ? skb->ifindex : v->oif;
TetherStatsValue* stat_v = bpf_tether_stats_map_lookup_elem(&stat_and_limit_k);
// If we don't have anywhere to put stats, then abort...
if (!stat_v) TC_PUNT(NO_STATS_ENTRY);
uint64_t* limit_v = bpf_tether_limit_map_lookup_elem(&stat_and_limit_k);
// If we don't have a limit, then abort...
if (!limit_v) TC_PUNT(NO_LIMIT_ENTRY);
// Required IPv6 minimum mtu is 1280, below that not clear what we should do, abort...
if (v->pmtu < IPV6_MIN_MTU) TC_PUNT(BELOW_IPV6_MTU);
// Approximate handling of TCP/IPv6 overhead for incoming LRO/GRO packets: default
// outbound path mtu of 1500 is not necessarily correct, but worst case we simply
// undercount, which is still better then not accounting for this overhead at all.
// Note: this really shouldn't be device/path mtu at all, but rather should be
// derived from this particular connection's mss (ie. from gro segment size).
// This would require a much newer kernel with newer ebpf accessors.
// (This is also blindly assuming 12 bytes of tcp timestamp option in tcp header)
uint64_t packets = 1;
uint64_t L3_bytes = skb->len - l2_header_size;
if (L3_bytes > v->pmtu) {
const int tcp6_overhead = sizeof(struct ipv6hdr) + sizeof(struct tcphdr) + 12;
const int mss = v->pmtu - tcp6_overhead;
const uint64_t payload = L3_bytes - tcp6_overhead;
packets = (payload + mss - 1) / mss;
L3_bytes = tcp6_overhead * packets + payload;
}
// Are we past the limit? If so, then abort...
// Note: will not overflow since u64 is 936 years even at 5Gbps.
// Do not drop here. Offload is just that, whenever we fail to handle
// a packet we let the core stack deal with things.
// (The core stack needs to handle limits correctly anyway,
// since we don't offload all traffic in both directions)
if (stat_v->rxBytes + stat_v->txBytes + L3_bytes > *limit_v) TC_PUNT(LIMIT_REACHED);
if (!is_ethernet) {
// Try to inject an ethernet header, and simply return if we fail.
// We do this even if TX interface is RAWIP and thus does not need an ethernet header,
// because this is easier and the kernel will strip extraneous ethernet header.
if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {
__sync_fetch_and_add(stream.down ? &stat_v->rxErrors : &stat_v->txErrors, 1);
TC_PUNT(CHANGE_HEAD_FAILED);
}
// bpf_skb_change_head() invalidates all pointers - reload them
data = (void*)(long)skb->data;
data_end = (void*)(long)skb->data_end;
eth = data;
ip6 = (void*)(eth + 1);
// I do not believe this can ever happen, but keep the verifier happy...
if (data + sizeof(struct ethhdr) + sizeof(*ip6) > data_end) {
__sync_fetch_and_add(stream.down ? &stat_v->rxErrors : &stat_v->txErrors, 1);
TC_DROP(TOO_SHORT);
}
};
// At this point we always have an ethernet header - which will get stripped by the
// kernel during transmit through a rawip interface. ie. 'eth' pointer is valid.
// Additionally note that 'is_ethernet' and 'l2_header_size' are no longer correct.
// CHECKSUM_COMPLETE is a 16-bit one's complement sum,
// thus corrections for it need to be done in 16-byte chunks at even offsets.
// IPv6 nexthdr is at offset 6, while hop limit is at offset 7
uint8_t old_hl = ip6->hop_limit;
--ip6->hop_limit;
uint8_t new_hl = ip6->hop_limit;
// bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
// (-ENOTSUPP) if it isn't.
bpf_csum_update(skb, 0xFFFF - ntohs(old_hl) + ntohs(new_hl));
__sync_fetch_and_add(stream.down ? &stat_v->rxPackets : &stat_v->txPackets, packets);
__sync_fetch_and_add(stream.down ? &stat_v->rxBytes : &stat_v->txBytes, L3_bytes);
// Overwrite any mac header with the new one
// For a rawip tx interface it will simply be a bunch of zeroes and later stripped.
*eth = v->macHeader;
// Redirect to forwarded interface.
//
// Note that bpf_redirect() cannot fail unless you pass invalid flags.
// The redirect actually happens after the ebpf program has already terminated,
// and can fail for example for mtu reasons at that point in time, but there's nothing
// we can do about it here.
return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
}
DEFINE_BPF_PROG("schedcls/tether_downstream6_ether", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream6_ether)
(struct __sk_buff* skb) {
return do_forward6(skb, ETHER, DOWNSTREAM, KVER_NONE);
}
DEFINE_BPF_PROG("schedcls/tether_upstream6_ether", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream6_ether)
(struct __sk_buff* skb) {
return do_forward6(skb, ETHER, UPSTREAM, KVER_NONE);
}
// Note: section names must be unique to prevent programs from appending to each other,
// so instead the bpf loader will strip everything past the final $ symbol when actually
// pinning the program into the filesystem.
//
// bpf_skb_change_head() is only present on 4.14+ and 2 trivial kernel patches are needed:
// ANDROID: net: bpf: Allow TC programs to call BPF_FUNC_skb_change_head
// ANDROID: net: bpf: permit redirect from ingress L3 to egress L2 devices at near max mtu
// (the first of those has already been upstreamed)
//
// These were added to 4.14+ Android Common Kernel in R (including the original release of ACK 5.4)
// and there is a test in kernel/tests/net/test/bpf_test.py testSkbChangeHead()
// and in system/netd/tests/binder_test.cpp NetdBinderTest TetherOffloadForwarding.
//
// Hence, these mandatory (must load successfully) implementations for 4.14+ kernels:
DEFINE_BPF_PROG_KVER("schedcls/tether_downstream6_rawip$4_14", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream6_rawip_4_14, KVER_4_14)
(struct __sk_buff* skb) {
return do_forward6(skb, RAWIP, DOWNSTREAM, KVER_4_14);
}
DEFINE_BPF_PROG_KVER("schedcls/tether_upstream6_rawip$4_14", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream6_rawip_4_14, KVER_4_14)
(struct __sk_buff* skb) {
return do_forward6(skb, RAWIP, UPSTREAM, KVER_4_14);
}
// and define no-op stubs for pre-4.14 kernels.
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream6_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream6_rawip_stub, KVER_NONE, KVER_4_14)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream6_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream6_rawip_stub, KVER_NONE, KVER_4_14)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
// ----- IPv4 Support -----
DEFINE_BPF_MAP_GRW(tether_downstream4_map, HASH, Tether4Key, Tether4Value, 1024, AID_NETWORK_STACK)
DEFINE_BPF_MAP_GRW(tether_upstream4_map, HASH, Tether4Key, Tether4Value, 1024, AID_NETWORK_STACK)
static inline __always_inline int do_forward4_bottom(struct __sk_buff* skb,
const int l2_header_size, void* data, const void* data_end,
struct ethhdr* eth, struct iphdr* ip, const struct rawip_bool rawip,
const struct stream_bool stream, const struct updatetime_bool updatetime,
const bool is_tcp, __unused const struct kver_uint kver) {
const bool is_ethernet = !rawip.rawip;
struct tcphdr* tcph = is_tcp ? (void*)(ip + 1) : NULL;
struct udphdr* udph = is_tcp ? NULL : (void*)(ip + 1);
if (is_tcp) {
// Make sure we can get at the tcp header
if (data + l2_header_size + sizeof(*ip) + sizeof(*tcph) > data_end)
TC_PUNT(SHORT_TCP_HEADER);
// If hardware offload is running and programming flows based on conntrack entries, try not
// to interfere with it, so do not offload TCP packets with any one of the SYN/FIN/RST flags
if (tcph->syn || tcph->fin || tcph->rst) TC_PUNT(TCPV4_CONTROL_PACKET);
} else { // UDP
// Make sure we can get at the udp header
if (data + l2_header_size + sizeof(*ip) + sizeof(*udph) > data_end)
TC_PUNT(SHORT_UDP_HEADER);
// Skip handling of CHECKSUM_COMPLETE packets with udp checksum zero due to need for
// additional updating of skb->csum (this could be fixed up manually with more effort).
//
// Note that the in-kernel implementation of 'int64_t bpf_csum_update(skb, u32 csum)' is:
// if (skb->ip_summed == CHECKSUM_COMPLETE)
// return (skb->csum = csum_add(skb->csum, csum));
// else
// return -ENOTSUPP;
//
// So this will punt any CHECKSUM_COMPLETE packet with a zero UDP checksum,
// and leave all other packets unaffected (since it just at most adds zero to skb->csum).
//
// In practice this should almost never trigger because most nics do not generate
// CHECKSUM_COMPLETE packets on receive - especially so for nics/drivers on a phone.
//
// Additionally since we're forwarding, in most cases the value of the skb->csum field
// shouldn't matter (it's not used by physical nic egress).
//
// It only matters if we're ingressing through a CHECKSUM_COMPLETE capable nic
// and egressing through a virtual interface looping back to the kernel itself
// (ie. something like veth) where the CHECKSUM_COMPLETE/skb->csum can get reused
// on ingress.
//
// If we were in the kernel we'd simply probably call
// void skb_checksum_complete_unset(struct sk_buff *skb) {
// if (skb->ip_summed == CHECKSUM_COMPLETE) skb->ip_summed = CHECKSUM_NONE;
// }
// here instead. Perhaps there should be a bpf helper for that?
if (!udph->check && (bpf_csum_update(skb, 0) >= 0)) TC_PUNT(UDP_CSUM_ZERO);
}
Tether4Key k = {
.iif = skb->ifindex,
.l4Proto = ip->protocol,
.src4.s_addr = ip->saddr,
.dst4.s_addr = ip->daddr,
.srcPort = is_tcp ? tcph->source : udph->source,
.dstPort = is_tcp ? tcph->dest : udph->dest,
};
if (is_ethernet) __builtin_memcpy(k.dstMac, eth->h_dest, ETH_ALEN);
Tether4Value* v = stream.down ? bpf_tether_downstream4_map_lookup_elem(&k)
: bpf_tether_upstream4_map_lookup_elem(&k);
// If we don't find any offload information then simply let the core stack handle it...
if (!v) return TC_ACT_PIPE;
uint32_t stat_and_limit_k = stream.down ? skb->ifindex : v->oif;
TetherStatsValue* stat_v = bpf_tether_stats_map_lookup_elem(&stat_and_limit_k);
// If we don't have anywhere to put stats, then abort...
if (!stat_v) TC_PUNT(NO_STATS_ENTRY);
uint64_t* limit_v = bpf_tether_limit_map_lookup_elem(&stat_and_limit_k);
// If we don't have a limit, then abort...
if (!limit_v) TC_PUNT(NO_LIMIT_ENTRY);
// Required IPv4 minimum mtu is 68, below that not clear what we should do, abort...
if (v->pmtu < 68) TC_PUNT(BELOW_IPV4_MTU);
// Approximate handling of TCP/IPv4 overhead for incoming LRO/GRO packets: default
// outbound path mtu of 1500 is not necessarily correct, but worst case we simply
// undercount, which is still better then not accounting for this overhead at all.
// Note: this really shouldn't be device/path mtu at all, but rather should be
// derived from this particular connection's mss (ie. from gro segment size).
// This would require a much newer kernel with newer ebpf accessors.
// (This is also blindly assuming 12 bytes of tcp timestamp option in tcp header)
uint64_t packets = 1;
uint64_t L3_bytes = skb->len - l2_header_size;
if (L3_bytes > v->pmtu) {
const int tcp4_overhead = sizeof(struct iphdr) + sizeof(struct tcphdr) + 12;
const int mss = v->pmtu - tcp4_overhead;
const uint64_t payload = L3_bytes - tcp4_overhead;
packets = (payload + mss - 1) / mss;
L3_bytes = tcp4_overhead * packets + payload;
}
// Are we past the limit? If so, then abort...
// Note: will not overflow since u64 is 936 years even at 5Gbps.
// Do not drop here. Offload is just that, whenever we fail to handle
// a packet we let the core stack deal with things.
// (The core stack needs to handle limits correctly anyway,
// since we don't offload all traffic in both directions)
if (stat_v->rxBytes + stat_v->txBytes + L3_bytes > *limit_v) TC_PUNT(LIMIT_REACHED);
if (!is_ethernet) {
// Try to inject an ethernet header, and simply return if we fail.
// We do this even if TX interface is RAWIP and thus does not need an ethernet header,
// because this is easier and the kernel will strip extraneous ethernet header.
if (bpf_skb_change_head(skb, sizeof(struct ethhdr), /*flags*/ 0)) {
__sync_fetch_and_add(stream.down ? &stat_v->rxErrors : &stat_v->txErrors, 1);
TC_PUNT(CHANGE_HEAD_FAILED);
}
// bpf_skb_change_head() invalidates all pointers - reload them
data = (void*)(long)skb->data;
data_end = (void*)(long)skb->data_end;
eth = data;
ip = (void*)(eth + 1);
tcph = is_tcp ? (void*)(ip + 1) : NULL;
udph = is_tcp ? NULL : (void*)(ip + 1);
// I do not believe this can ever happen, but keep the verifier happy...
if (data + sizeof(struct ethhdr) + sizeof(*ip) + (is_tcp ? sizeof(*tcph) : sizeof(*udph)) > data_end) {
__sync_fetch_and_add(stream.down ? &stat_v->rxErrors : &stat_v->txErrors, 1);
TC_DROP(TOO_SHORT);
}
};
// At this point we always have an ethernet header - which will get stripped by the
// kernel during transmit through a rawip interface. ie. 'eth' pointer is valid.
// Additionally note that 'is_ethernet' and 'l2_header_size' are no longer correct.
// Overwrite any mac header with the new one
// For a rawip tx interface it will simply be a bunch of zeroes and later stripped.
*eth = v->macHeader;
// Decrement the IPv4 TTL, we already know it's greater than 1.
// u8 TTL field is followed by u8 protocol to make a u16 for ipv4 header checksum update.
// Since we're keeping the ipv4 checksum valid (which means the checksum of the entire
// ipv4 header remains 0), the overall checksum of the entire packet does not change.
const int sz2 = sizeof(__be16);
const __be16 old_ttl_proto = *(__be16 *)&ip->ttl;
const __be16 new_ttl_proto = old_ttl_proto - htons(0x0100);
bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_ttl_proto, new_ttl_proto, sz2);
bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(ttl), &new_ttl_proto, sz2, 0);
const int l4_offs_csum = is_tcp ? ETH_IP4_TCP_OFFSET(check) : ETH_IP4_UDP_OFFSET(check);
const int sz4 = sizeof(__be32);
// UDP 0 is special and stored as FFFF (this flag also causes a csum of 0 to be unmodified)
const int l4_flags = is_tcp ? 0 : BPF_F_MARK_MANGLED_0;
const __be32 old_daddr = k.dst4.s_addr;
const __be32 old_saddr = k.src4.s_addr;
const __be32 new_daddr = v->dst46.s6_addr32[3];
const __be32 new_saddr = v->src46.s6_addr32[3];
bpf_l4_csum_replace(skb, l4_offs_csum, old_daddr, new_daddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);
bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_daddr, new_daddr, sz4);
bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(daddr), &new_daddr, sz4, 0);
bpf_l4_csum_replace(skb, l4_offs_csum, old_saddr, new_saddr, sz4 | BPF_F_PSEUDO_HDR | l4_flags);
bpf_l3_csum_replace(skb, ETH_IP4_OFFSET(check), old_saddr, new_saddr, sz4);
bpf_skb_store_bytes(skb, ETH_IP4_OFFSET(saddr), &new_saddr, sz4, 0);
// The offsets for TCP and UDP ports: source (u16 @ L4 offset 0) & dest (u16 @ L4 offset 2) are
// actually the same, so the compiler should just optimize them both down to a constant.
bpf_l4_csum_replace(skb, l4_offs_csum, k.srcPort, v->srcPort, sz2 | l4_flags);
bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(source) : ETH_IP4_UDP_OFFSET(source),
&v->srcPort, sz2, 0);
bpf_l4_csum_replace(skb, l4_offs_csum, k.dstPort, v->dstPort, sz2 | l4_flags);
bpf_skb_store_bytes(skb, is_tcp ? ETH_IP4_TCP_OFFSET(dest) : ETH_IP4_UDP_OFFSET(dest),
&v->dstPort, sz2, 0);
// This requires the bpf_ktime_get_boot_ns() helper which was added in 5.8,
// and backported to all Android Common Kernel 4.14+ trees.
if (updatetime.updatetime) v->last_used = bpf_ktime_get_boot_ns();
__sync_fetch_and_add(stream.down ? &stat_v->rxPackets : &stat_v->txPackets, packets);
__sync_fetch_and_add(stream.down ? &stat_v->rxBytes : &stat_v->txBytes, L3_bytes);
// Redirect to forwarded interface.
//
// Note that bpf_redirect() cannot fail unless you pass invalid flags.
// The redirect actually happens after the ebpf program has already terminated,
// and can fail for example for mtu reasons at that point in time, but there's nothing
// we can do about it here.
return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
}
static inline __always_inline int do_forward4(struct __sk_buff* skb,
const struct rawip_bool rawip,
const struct stream_bool stream,
const struct updatetime_bool updatetime,
const struct kver_uint kver) {
const bool is_ethernet = !rawip.rawip;
// Require ethernet dst mac address to be our unicast address.
if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;
// Must be meta-ethernet IPv4 frame
if (skb->protocol != htons(ETH_P_IP)) return TC_ACT_PIPE;
const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
// Since the program never writes via DPA (direct packet access) auto-pull/unclone logic does
// not trigger and thus we need to manually make sure we can read packet headers via DPA.
// Note: this is a blind best effort pull, which may fail or pull less - this doesn't matter.
// It has to be done early cause it will invalidate any skb->data/data_end derived pointers.
try_make_writable(skb, l2_header_size + IP4_HLEN + TCP_HLEN);
void* data = (void*)(long)skb->data;
const void* data_end = (void*)(long)skb->data_end;
struct ethhdr* eth = is_ethernet ? data : NULL; // used iff is_ethernet
struct iphdr* ip = is_ethernet ? (void*)(eth + 1) : data;
// Must have (ethernet and) ipv4 header
if (data + l2_header_size + sizeof(*ip) > data_end) return TC_ACT_PIPE;
// Ethertype - if present - must be IPv4
if (is_ethernet && (eth->h_proto != htons(ETH_P_IP))) return TC_ACT_PIPE;
// IP version must be 4
if (ip->version != 4) TC_PUNT(INVALID_IPV4_VERSION);
// We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
if (ip->ihl != 5) TC_PUNT(HAS_IP_OPTIONS);
// Calculate the IPv4 one's complement checksum of the IPv4 header.
__wsum sum4 = 0;
for (unsigned i = 0; i < sizeof(*ip) / sizeof(__u16); ++i) {
sum4 += ((__u16*)ip)[i];
}
// Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse u32 into range 1 .. 0x1FFFE
sum4 = (sum4 & 0xFFFF) + (sum4 >> 16); // collapse any potential carry into u16
// for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
if (sum4 != 0xFFFF) TC_PUNT(CHECKSUM);
// Minimum IPv4 total length is the size of the header
if (ntohs(ip->tot_len) < sizeof(*ip)) TC_PUNT(TRUNCATED_IPV4);
// We are incapable of dealing with IPv4 fragments
if (ip->frag_off & ~htons(IP_DF)) TC_PUNT(IS_IP_FRAG);
// Cannot decrement during forward if already zero or would be zero,
// Let the kernel's stack handle these cases and generate appropriate ICMP errors.
if (ip->ttl <= 1) TC_PUNT(LOW_TTL);
// If we cannot update the 'last_used' field due to lack of bpf_ktime_get_boot_ns() helper,
// then it is not safe to offload UDP due to the small conntrack timeouts, as such,
// in such a situation we can only support TCP. This also has the added nice benefit of
// using a separate error counter, and thus making it obvious which version of the program
// is loaded.
if (!updatetime.updatetime && ip->protocol != IPPROTO_TCP) TC_PUNT(NON_TCP);
// We do not support offloading anything besides IPv4 TCP and UDP, due to need for NAT,
// but no need to check this if !updatetime due to check immediately above.
if (updatetime.updatetime && (ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP))
TC_PUNT(NON_TCP_UDP);
// We want to make sure that the compiler will, in the !updatetime case, entirely optimize
// out all the non-tcp logic. Also note that at this point is_udp === !is_tcp.
const bool is_tcp = !updatetime.updatetime || (ip->protocol == IPPROTO_TCP);
// This is a bit of a hack to make things easier on the bpf verifier.
// (In particular I believe the Linux 4.14 kernel's verifier can get confused later on about
// what offsets into the packet are valid and can spuriously reject the program, this is
// because it fails to realize that is_tcp && !is_tcp is impossible)
//
// For both TCP & UDP we'll need to read and modify the src/dst ports, which so happen to
// always be in the first 4 bytes of the L4 header. Additionally for UDP we'll need access
// to the checksum field which is in bytes 7 and 8. While for TCP we'll need to read the
// TCP flags (at offset 13) and access to the checksum field (2 bytes at offset 16).
// As such we *always* need access to at least 8 bytes.
if (data + l2_header_size + sizeof(*ip) + 8 > data_end) TC_PUNT(SHORT_L4_HEADER);
// We're forcing the compiler to emit two copies of the following code, optimized
// separately for is_tcp being true or false. This simplifies the resulting bpf
// byte code sufficiently that the 4.14 bpf verifier is able to keep track of things.
// Without this (updatetime == true) case would fail to bpf verify on 4.14 even
// if the underlying requisite kernel support (bpf_ktime_get_boot_ns) was backported.
if (is_tcp) {
return do_forward4_bottom(skb, l2_header_size, data, data_end, eth, ip,
rawip, stream, updatetime, /* is_tcp */ true, kver);
} else {
return do_forward4_bottom(skb, l2_header_size, data, data_end, eth, ip,
rawip, stream, updatetime, /* is_tcp */ false, kver);
}
}
// Full featured (required) implementations for 5.8+ kernels (these are S+ by definition)
DEFINE_BPF_PROG_KVER("schedcls/tether_downstream4_rawip$5_8", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_rawip_5_8, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, DOWNSTREAM, UPDATETIME, KVER_5_8);
}
DEFINE_BPF_PROG_KVER("schedcls/tether_upstream4_rawip$5_8", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_rawip_5_8, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, UPSTREAM, UPDATETIME, KVER_5_8);
}
DEFINE_BPF_PROG_KVER("schedcls/tether_downstream4_ether$5_8", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_ether_5_8, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, DOWNSTREAM, UPDATETIME, KVER_5_8);
}
DEFINE_BPF_PROG_KVER("schedcls/tether_upstream4_ether$5_8", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_ether_5_8, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, UPSTREAM, UPDATETIME, KVER_5_8);
}
// Full featured (optional) implementations for 4.14-S, 4.19-S & 5.4-S kernels
// (optional, because we need to be able to fallback for 4.14/4.19/5.4 pre-S kernels)
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$opt",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_rawip_opt,
KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, DOWNSTREAM, UPDATETIME, KVER_4_14);
}
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$opt",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_rawip_opt,
KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, UPSTREAM, UPDATETIME, KVER_4_14);
}
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$opt",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_ether_opt,
KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, DOWNSTREAM, UPDATETIME, KVER_4_14);
}
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$opt",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_ether_opt,
KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, UPSTREAM, UPDATETIME, KVER_4_14);
}
// Partial (TCP-only: will not update 'last_used' field) implementations for 4.14+ kernels.
// These will be loaded only if the above optional ones failed (loading of *these* must succeed
// for 5.4+, since that is always an R patched kernel).
//
// [Note: as a result TCP connections will not have their conntrack timeout refreshed, however,
// since /proc/sys/net/netfilter/nf_conntrack_tcp_timeout_established defaults to 432000 (seconds),
// this in practice means they'll break only after 5 days. This seems an acceptable trade-off.
//
// Additionally kernel/tests change "net-test: add bpf_ktime_get_ns / bpf_ktime_get_boot_ns tests"
// which enforces and documents the required kernel cherrypicks will make it pretty unlikely that
// many devices upgrading to S will end up relying on these fallback programs.
// RAWIP: Required for 5.4-R kernels -- which always support bpf_skb_change_head().
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_rawip_5_4, KVER_5_4, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, DOWNSTREAM, NO_UPDATETIME, KVER_5_4);
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$5_4", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_rawip_5_4, KVER_5_4, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, UPSTREAM, NO_UPDATETIME, KVER_5_4);
}
// RAWIP: Optional for 4.14/4.19 (R) kernels -- which support bpf_skb_change_head().
// [Note: fallback for 4.14/4.19 (P/Q) kernels is below in stub section]
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$4_14",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_rawip_4_14,
KVER_4_14, KVER_5_4)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, DOWNSTREAM, NO_UPDATETIME, KVER_4_14);
}
DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$4_14",
AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_rawip_4_14,
KVER_4_14, KVER_5_4)
(struct __sk_buff* skb) {
return do_forward4(skb, RAWIP, UPSTREAM, NO_UPDATETIME, KVER_4_14);
}
// ETHER: Required for 4.14-Q/R, 4.19-Q/R & 5.4-R kernels.
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$4_14", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_ether_4_14, KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, DOWNSTREAM, NO_UPDATETIME, KVER_4_14);
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$4_14", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_ether_4_14, KVER_4_14, KVER_5_8)
(struct __sk_buff* skb) {
return do_forward4(skb, ETHER, UPSTREAM, NO_UPDATETIME, KVER_4_14);
}
// Placeholder (no-op) implementations for older Q kernels
// RAWIP: 4.9-P/Q, 4.14-P/Q & 4.19-Q kernels -- without bpf_skb_change_head() for tc programs
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_rawip_stub, KVER_NONE, KVER_5_4)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_rawip$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_rawip_stub, KVER_NONE, KVER_5_4)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
// ETHER: 4.9-P/Q kernel
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_downstream4_ether$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_downstream4_ether_stub, KVER_NONE, KVER_4_14)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
DEFINE_BPF_PROG_KVER_RANGE("schedcls/tether_upstream4_ether$stub", AID_ROOT, AID_NETWORK_STACK,
sched_cls_tether_upstream4_ether_stub, KVER_NONE, KVER_4_14)
(__unused struct __sk_buff* skb) {
return TC_ACT_PIPE;
}
// ----- XDP Support -----
DEFINE_BPF_MAP_GRW(tether_dev_map, DEVMAP_HASH, uint32_t, uint32_t, 64, AID_NETWORK_STACK)
static inline __always_inline int do_xdp_forward6(__unused struct xdp_md *ctx,
__unused const struct rawip_bool rawip, __unused const struct stream_bool stream) {
return XDP_PASS;
}
static inline __always_inline int do_xdp_forward4(__unused struct xdp_md *ctx,
__unused const struct rawip_bool rawip, __unused const struct stream_bool stream) {
return XDP_PASS;
}
static inline __always_inline int do_xdp_forward_ether(struct xdp_md *ctx,
const struct stream_bool stream) {
const void* data = (void*)(long)ctx->data;
const void* data_end = (void*)(long)ctx->data_end;
const struct ethhdr* eth = data;
// Make sure we actually have an ethernet header
if ((void*)(eth + 1) > data_end) return XDP_PASS;
if (eth->h_proto == htons(ETH_P_IPV6))
return do_xdp_forward6(ctx, ETHER, stream);
if (eth->h_proto == htons(ETH_P_IP))
return do_xdp_forward4(ctx, ETHER, stream);
// Anything else we don't know how to handle...
return XDP_PASS;
}
static inline __always_inline int do_xdp_forward_rawip(struct xdp_md *ctx,
const struct stream_bool stream) {
const void* data = (void*)(long)ctx->data;
const void* data_end = (void*)(long)ctx->data_end;
// The top nibble of both IPv4 and IPv6 headers is the IP version.
if (data_end - data < 1) return XDP_PASS;
const uint8_t v = (*(uint8_t*)data) >> 4;
if (v == 6) return do_xdp_forward6(ctx, RAWIP, stream);
if (v == 4) return do_xdp_forward4(ctx, RAWIP, stream);
// Anything else we don't know how to handle...
return XDP_PASS;
}
#define DEFINE_XDP_PROG(str, func) \
DEFINE_BPF_PROG_KVER(str, AID_ROOT, AID_NETWORK_STACK, func, KVER_5_9)(struct xdp_md *ctx)
DEFINE_XDP_PROG("xdp/tether_downstream_ether",
xdp_tether_downstream_ether) {
return do_xdp_forward_ether(ctx, DOWNSTREAM);
}
DEFINE_XDP_PROG("xdp/tether_downstream_rawip",
xdp_tether_downstream_rawip) {
return do_xdp_forward_rawip(ctx, DOWNSTREAM);
}
DEFINE_XDP_PROG("xdp/tether_upstream_ether",
xdp_tether_upstream_ether) {
return do_xdp_forward_ether(ctx, UPSTREAM);
}
DEFINE_XDP_PROG("xdp/tether_upstream_rawip",
xdp_tether_upstream_rawip) {
return do_xdp_forward_rawip(ctx, UPSTREAM);
}
LICENSE("Apache 2.0");
CRITICAL("Connectivity (Tethering)");

166
bpf/progs/offload.h Normal file
View file

@ -0,0 +1,166 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <linux/if.h>
#include <linux/if_ether.h>
#include <linux/in.h>
#include <linux/in6.h>
// Common definitions for BPF code in the tethering mainline module.
// These definitions are available to:
// - The BPF programs in Tethering/bpf_progs/
// - JNI code that depends on the bpf_connectivity_headers library.
#define BPF_TETHER_ERRORS \
ERR(INVALID_IPV4_VERSION) \
ERR(INVALID_IPV6_VERSION) \
ERR(LOW_TTL) \
ERR(INVALID_TCP_HEADER) \
ERR(TCPV4_CONTROL_PACKET) \
ERR(TCPV6_CONTROL_PACKET) \
ERR(NON_GLOBAL_SRC) \
ERR(NON_GLOBAL_DST) \
ERR(LOCAL_SRC_DST) \
ERR(NO_STATS_ENTRY) \
ERR(NO_LIMIT_ENTRY) \
ERR(BELOW_IPV4_MTU) \
ERR(BELOW_IPV6_MTU) \
ERR(LIMIT_REACHED) \
ERR(CHANGE_HEAD_FAILED) \
ERR(TOO_SHORT) \
ERR(HAS_IP_OPTIONS) \
ERR(IS_IP_FRAG) \
ERR(CHECKSUM) \
ERR(NON_TCP_UDP) \
ERR(NON_TCP) \
ERR(SHORT_L4_HEADER) \
ERR(SHORT_TCP_HEADER) \
ERR(SHORT_UDP_HEADER) \
ERR(UDP_CSUM_ZERO) \
ERR(TRUNCATED_IPV4) \
ERR(_MAX)
#define ERR(x) BPF_TETHER_ERR_ ##x,
enum {
BPF_TETHER_ERRORS
};
#undef ERR
#define ERR(x) #x,
static const char *bpf_tether_errors[] = {
BPF_TETHER_ERRORS
};
#undef ERR
// This header file is shared by eBPF kernel programs (C) and netd (C++) and
// some of the maps are also accessed directly from Java mainline module code.
//
// Hence: explicitly pad all relevant structures and assert that their size
// is the sum of the sizes of their fields.
#define STRUCT_SIZE(name, size) _Static_assert(sizeof(name) == (size), "Incorrect struct size.")
typedef uint32_t TetherStatsKey; // upstream ifindex
typedef struct {
uint64_t rxPackets;
uint64_t rxBytes;
uint64_t rxErrors;
uint64_t txPackets;
uint64_t txBytes;
uint64_t txErrors;
} TetherStatsValue;
STRUCT_SIZE(TetherStatsValue, 6 * 8); // 48
typedef uint32_t TetherLimitKey; // upstream ifindex
typedef uint64_t TetherLimitValue; // in bytes
// For now tethering offload only needs to support downstreams that use 6-byte MAC addresses,
// because all downstream types that are currently supported (WiFi, USB, Bluetooth and
// Ethernet) have 6-byte MAC addresses.
typedef struct {
uint32_t iif; // The input interface index
uint8_t dstMac[ETH_ALEN]; // destination ethernet mac address (zeroed iff rawip ingress)
uint8_t zero[2]; // zero pad for 8 byte alignment
struct in6_addr neigh6; // The destination IPv6 address
} TetherDownstream6Key;
STRUCT_SIZE(TetherDownstream6Key, 4 + 6 + 2 + 16); // 28
typedef struct {
uint32_t oif; // The output interface to redirect to
struct ethhdr macHeader; // includes dst/src mac and ethertype (zeroed iff rawip egress)
uint16_t pmtu; // The maximum L3 output path/route mtu
} Tether6Value;
STRUCT_SIZE(Tether6Value, 4 + 14 + 2); // 20
typedef struct {
uint32_t iif; // The input interface index
uint8_t dstMac[ETH_ALEN]; // destination ethernet mac address (zeroed iff rawip ingress)
uint16_t l4Proto; // IPPROTO_TCP/UDP/...
struct in6_addr src6; // source &
struct in6_addr dst6; // destination IPv6 addresses
__be16 srcPort; // source &
__be16 dstPort; // destination tcp/udp/... ports
} TetherDownstream64Key;
STRUCT_SIZE(TetherDownstream64Key, 4 + 6 + 2 + 16 + 16 + 2 + 2); // 48
typedef struct {
uint32_t oif; // The output interface to redirect to
struct ethhdr macHeader; // includes dst/src mac and ethertype (zeroed iff rawip egress)
uint16_t pmtu; // The maximum L3 output path/route mtu
struct in_addr src4; // source &
struct in_addr dst4; // destination IPv4 addresses
__be16 srcPort; // source &
__be16 outPort; // destination tcp/udp/... ports
uint64_t lastUsed; // Kernel updates on each use with bpf_ktime_get_boot_ns()
} TetherDownstream64Value;
STRUCT_SIZE(TetherDownstream64Value, 4 + 14 + 2 + 4 + 4 + 2 + 2 + 8); // 40
typedef struct {
uint32_t iif; // The input interface index
uint8_t dstMac[ETH_ALEN]; // destination ethernet mac address (zeroed iff rawip ingress)
uint8_t zero[6]; // zero pad for 8 byte alignment
uint64_t src64; // Top 64-bits of the src ip
} TetherUpstream6Key;
STRUCT_SIZE(TetherUpstream6Key, 4 + 6 + 6 + 8); // 24
typedef struct {
uint32_t iif; // The input interface index
uint8_t dstMac[ETH_ALEN]; // destination ethernet mac address (zeroed iff rawip ingress)
uint16_t l4Proto; // IPPROTO_TCP/UDP/...
struct in_addr src4; // source &
struct in_addr dst4; // destination IPv4 addresses
__be16 srcPort; // source &
__be16 dstPort; // destination TCP/UDP/... ports
} Tether4Key;
STRUCT_SIZE(Tether4Key, 4 + 6 + 2 + 4 + 4 + 2 + 2); // 24
typedef struct {
uint32_t oif; // The output interface to redirect to
struct ethhdr macHeader; // includes dst/src mac and ethertype (zeroed iff rawip egress)
uint16_t pmtu; // Maximum L3 output path/route mtu
struct in6_addr src46; // source & (always IPv4 mapped for downstream)
struct in6_addr dst46; // destination IP addresses (may be IPv4 mapped or IPv6 for upstream)
__be16 srcPort; // source &
__be16 dstPort; // destination tcp/udp/... ports
uint64_t last_used; // Kernel updates on each use with bpf_ktime_get_boot_ns()
} Tether4Value;
STRUCT_SIZE(Tether4Value, 4 + 14 + 2 + 16 + 16 + 2 + 2 + 8); // 64
#undef STRUCT_SIZE

1
bpf/progs/offload@mainline.c Symbolic link
View file

@ -0,0 +1 @@
offload.c

63
bpf/progs/test.c Normal file
View file

@ -0,0 +1,63 @@
/*
* Copyright (C) 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef MAINLINE
// BTF is incompatible with bpfloaders < v0.10, hence for S (v0.2) we must
// ship a different file than for later versions, but we need bpfloader v0.25+
// for obj@ver.o support
#define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
#else /* MAINLINE */
// The resulting .o needs to load on the Android S bpfloader
#define BPFLOADER_MIN_VER BPFLOADER_S_VERSION
#define BPFLOADER_MAX_VER BPFLOADER_T_VERSION
#endif /* MAINLINE */
// This is non production code, only used for testing
// Needed because the bitmap array definition is non-kosher for pre-T OS devices.
#define THIS_BPF_PROGRAM_IS_FOR_TEST_PURPOSES_ONLY
#include "bpf_net_helpers.h"
#include "offload.h"
// Used only by TetheringPrivilegedTests, not by production code.
DEFINE_BPF_MAP_GRW(tether_downstream6_map, HASH, TetherDownstream6Key, Tether6Value, 16,
AID_NETWORK_STACK)
DEFINE_BPF_MAP_GRW(tether2_downstream6_map, HASH, TetherDownstream6Key, Tether6Value, 16,
AID_NETWORK_STACK)
DEFINE_BPF_MAP_GRW(tether3_downstream6_map, HASH, TetherDownstream6Key, Tether6Value, 16,
AID_NETWORK_STACK)
// Used only by BpfBitmapTest, not by production code.
DEFINE_BPF_MAP_GRW(bitmap, ARRAY, int, uint64_t, 2, AID_NETWORK_STACK)
DEFINE_BPF_PROG_KVER("xdp/drop_ipv4_udp_ether", AID_ROOT, AID_NETWORK_STACK,
xdp_test, KVER_5_9)
(struct xdp_md *ctx) {
void *data = (void *)(long)ctx->data;
void *data_end = (void *)(long)ctx->data_end;
struct ethhdr *eth = data;
int hsize = sizeof(*eth);
struct iphdr *ip = data + hsize;
hsize += sizeof(struct iphdr);
if (data + hsize > data_end) return XDP_PASS;
if (eth->h_proto != htons(ETH_P_IP)) return XDP_PASS;
if (ip->protocol == IPPROTO_UDP) return XDP_DROP;
return XDP_PASS;
}
LICENSE("Apache 2.0");

1
bpf/progs/test@mainline.c Symbolic link
View file

@ -0,0 +1 @@
test.c