From 732a14164fbfa417b433cc7f6545b5542dcfa523 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Maciej=20=C5=BBenczykowski?= <maze@google.com>
Date: Thu, 14 Mar 2024 00:17:18 -0700
Subject: [PATCH] NetBpfLoad: rework mainline bpf execution codepath
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

(for now for better testing we don't make V special)

Test: TreeHugger
Signed-off-by: Maciej Żenczykowski <maze@google.com>
Change-Id: Id2835bd393c0ec9e1710644681bc5aab0d7d2aae
---
 netbpfload/NetBpfLoad.cpp                     | 25 ++++++++---------
 netbpfload/netbpfload.mainline.rc             | 10 ++++++-
 netd/BpfHandler.cpp                           | 27 +++++++++++++++++++
 ...id_server_connectivity_ClatCoordinator.cpp |  7 ++++-
 4 files changed, 55 insertions(+), 14 deletions(-)

diff --git a/netbpfload/NetBpfLoad.cpp b/netbpfload/NetBpfLoad.cpp
index 9651a3b29c..83bb98c875 100644
--- a/netbpfload/NetBpfLoad.cpp
+++ b/netbpfload/NetBpfLoad.cpp
@@ -257,12 +257,6 @@ int main(int argc, char** argv, char * const envp[]) {
 
     logTetheringApexVersion();
 
-    if (has_platform_bpfloader_rc && !has_platform_netbpfload_rc) {
-        // Tethering apex shipped initrc file causes us to reach here
-        // but we're not ready to correctly handle anything before U QPR2
-        // in which the 'bpfloader' vs 'netbpfload' split happened
-    }
-
     if (!isAtLeastT) {
         ALOGE("Impossible - not reachable on Android <T.");
         return 1;
@@ -319,14 +313,16 @@ int main(int argc, char** argv, char * const envp[]) {
         return 1;
     }
 
-    if (isAtLeastU) {
+    if (false && isAtLeastV) {
         // Linux 5.16-rc1 changed the default to 2 (disabled but changeable),
         // but we need 0 (enabled)
         // (this writeFile is known to fail on at least 4.19, but always defaults to 0 on
         // pre-5.13, on 5.13+ it depends on CONFIG_BPF_UNPRIV_DEFAULT_OFF)
         if (writeProcSysFile("/proc/sys/kernel/unprivileged_bpf_disabled", "0\n") &&
             android::bpf::isAtLeastKernelVersion(5, 13, 0)) return 1;
+    }
 
+    if (isAtLeastU) {
         // Enable the eBPF JIT -- but do note that on 64-bit kernels it is likely
         // already force enabled by the kernel config option BPF_JIT_ALWAYS_ON.
         // (Note: this (open) will fail with ENOENT 'No such file or directory' if
@@ -384,10 +380,15 @@ int main(int argc, char** argv, char * const envp[]) {
         return 1;
     }
 
-    ALOGI("done, transferring control to platform bpfloader.");
+    if (false && isAtLeastV) {
+        ALOGI("done, transferring control to platform bpfloader.");
 
-    const char * args[] = { platformBpfLoader, NULL, };
-    execve(args[0], (char**)args, envp);
-    ALOGE("FATAL: execve('%s'): %d[%s]", platformBpfLoader, errno, strerror(errno));
-    return 1;
+        const char * args[] = { platformBpfLoader, NULL, };
+        execve(args[0], (char**)args, envp);
+        ALOGE("FATAL: execve('%s'): %d[%s]", platformBpfLoader, errno, strerror(errno));
+        return 1;
+    }
+
+    ALOGI("mainline done!");
+    return 0;
 }
diff --git a/netbpfload/netbpfload.mainline.rc b/netbpfload/netbpfload.mainline.rc
index 0ac5de8b97..d7202f7ba7 100644
--- a/netbpfload/netbpfload.mainline.rc
+++ b/netbpfload/netbpfload.mainline.rc
@@ -1,8 +1,16 @@
-service bpfloader /apex/com.android.tethering/bin/netbpfload
+service mdnsd_loadbpf /system/bin/bpfloader
     capabilities CHOWN SYS_ADMIN NET_ADMIN
     group root graphics network_stack net_admin net_bw_acct net_bw_stats net_raw system
     user root
     rlimit memlock 1073741824 1073741824
     oneshot
     reboot_on_failure reboot,bpfloader-failed
+
+service bpfloader /apex/com.android.tethering/bin/netbpfload
+    capabilities CHOWN SYS_ADMIN NET_ADMIN
+    group system root graphics network_stack net_admin net_bw_acct net_bw_stats net_raw
+    user system
+    rlimit memlock 1073741824 1073741824
+    oneshot
+    reboot_on_failure reboot,bpfloader-failed
     override
diff --git a/netd/BpfHandler.cpp b/netd/BpfHandler.cpp
index a00c363254..e6fc825822 100644
--- a/netd/BpfHandler.cpp
+++ b/netd/BpfHandler.cpp
@@ -165,8 +165,35 @@ BpfHandler::BpfHandler()
 BpfHandler::BpfHandler(uint32_t perUidLimit, uint32_t totalLimit)
     : mPerUidStatsEntriesLimit(perUidLimit), mTotalUidStatsEntriesLimit(totalLimit) {}
 
+// copied with minor changes from waitForProgsLoaded()
+// p/m/C's staticlibs/native/bpf_headers/include/bpf/WaitForProgsLoaded.h
+static inline void waitForNetProgsLoaded() {
+    // infinite loop until success with 5/10/20/40/60/60/60... delay
+    for (int delay = 5;; delay *= 2) {
+        if (delay > 60) delay = 60;
+        if (base::WaitForProperty("init.svc.bpfloader", "stopped", std::chrono::seconds(delay))
+            && !access("/sys/fs/bpf/netd_shared", F_OK))
+            return;
+        ALOGW("Waited %ds for init.svc.bpfloader=stopped, still waiting...", delay);
+    }
+}
+
 Status BpfHandler::init(const char* cg2_path) {
     // Make sure BPF programs are loaded before doing anything
+    ALOGI("Waiting for BPF programs");
+
+    if (true || !modules::sdklevel::IsAtLeastV()) {
+        waitForNetProgsLoaded();
+        ALOGI("Networking BPF programs are loaded");
+
+        if (!base::SetProperty("ctl.start", "mdnsd_loadbpf")) {
+            ALOGE("Failed to set property ctl.start=mdnsd_loadbpf, see dmesg for reason.");
+            abort();
+        }
+
+        ALOGI("Waiting for remaining BPF programs");
+    }
+
     android::bpf::waitForProgsLoaded();
     ALOGI("BPF programs are loaded");
 
diff --git a/service/jni/com_android_server_connectivity_ClatCoordinator.cpp b/service/jni/com_android_server_connectivity_ClatCoordinator.cpp
index c125bd6479..4214bc9a15 100644
--- a/service/jni/com_android_server_connectivity_ClatCoordinator.cpp
+++ b/service/jni/com_android_server_connectivity_ClatCoordinator.cpp
@@ -113,7 +113,12 @@ static void verifyClatPerms() {
     if (!modules::sdklevel::IsAtLeastT()) return;
 
     V("/sys/fs/bpf", S_IFDIR|S_ISVTX|0777, ROOT, ROOT, "fs_bpf", DIR);
-    V("/sys/fs/bpf/net_shared", S_IFDIR|S_ISVTX|0777, ROOT, ROOT, "fs_bpf_net_shared", DIR);
+
+    if (false && modules::sdklevel::IsAtLeastV()) {
+        V("/sys/fs/bpf/net_shared", S_IFDIR|01777, ROOT, ROOT, "fs_bpf_net_shared", DIR);
+    } else {
+        V("/sys/fs/bpf/net_shared", S_IFDIR|01777, SYSTEM, SYSTEM, "fs_bpf_net_shared", DIR);
+    }
 
     // pre-U we do not have selinux privs to getattr on bpf maps/progs
     // so while the below *should* be as listed, we have no way to actually verify