NetBpfLoad: rework mainline bpf execution codepath

(for now for better testing we don't make V special)

Test: TreeHugger
Signed-off-by: Maciej Żenczykowski <maze@google.com>
Change-Id: Id2835bd393c0ec9e1710644681bc5aab0d7d2aae

netbpfload/NetBpfLoad.cpp

@@ -257,12 +257,6 @@ int main(int argc, char** argv, char * const envp[]) {
 
     logTetheringApexVersion();
 
-    if (has_platform_bpfloader_rc && !has_platform_netbpfload_rc) {
-        // Tethering apex shipped initrc file causes us to reach here
-        // but we're not ready to correctly handle anything before U QPR2
-        // in which the 'bpfloader' vs 'netbpfload' split happened
-    }
-
     if (!isAtLeastT) {
         ALOGE("Impossible - not reachable on Android <T.");
         return 1;
@@ -319,14 +313,16 @@ int main(int argc, char** argv, char * const envp[]) {
         return 1;
     }
 
-    if (isAtLeastU) {
+    if (false && isAtLeastV) {
         // Linux 5.16-rc1 changed the default to 2 (disabled but changeable),
         // but we need 0 (enabled)
         // (this writeFile is known to fail on at least 4.19, but always defaults to 0 on
         // pre-5.13, on 5.13+ it depends on CONFIG_BPF_UNPRIV_DEFAULT_OFF)
         if (writeProcSysFile("/proc/sys/kernel/unprivileged_bpf_disabled", "0\n") &&
             android::bpf::isAtLeastKernelVersion(5, 13, 0)) return 1;
+    }
 
+    if (isAtLeastU) {
         // Enable the eBPF JIT -- but do note that on 64-bit kernels it is likely
         // already force enabled by the kernel config option BPF_JIT_ALWAYS_ON.
         // (Note: this (open) will fail with ENOENT 'No such file or directory' if
@@ -384,10 +380,15 @@ int main(int argc, char** argv, char * const envp[]) {
         return 1;
     }
 
-    ALOGI("done, transferring control to platform bpfloader.");
+    if (false && isAtLeastV) {
+        ALOGI("done, transferring control to platform bpfloader.");
 
-    const char * args[] = { platformBpfLoader, NULL, };
-    execve(args[0], (char**)args, envp);
-    ALOGE("FATAL: execve('%s'): %d[%s]", platformBpfLoader, errno, strerror(errno));
-    return 1;
+        const char * args[] = { platformBpfLoader, NULL, };
+        execve(args[0], (char**)args, envp);
+        ALOGE("FATAL: execve('%s'): %d[%s]", platformBpfLoader, errno, strerror(errno));
+        return 1;
+    }
+
+    ALOGI("mainline done!");
+    return 0;
 }

netbpfload/netbpfload.mainline.rc

@@ -1,8 +1,16 @@
-service bpfloader /apex/com.android.tethering/bin/netbpfload
+service mdnsd_loadbpf /system/bin/bpfloader
     capabilities CHOWN SYS_ADMIN NET_ADMIN
     group root graphics network_stack net_admin net_bw_acct net_bw_stats net_raw system
     user root
     rlimit memlock 1073741824 1073741824
     oneshot
     reboot_on_failure reboot,bpfloader-failed
+
+service bpfloader /apex/com.android.tethering/bin/netbpfload
+    capabilities CHOWN SYS_ADMIN NET_ADMIN
+    group system root graphics network_stack net_admin net_bw_acct net_bw_stats net_raw
+    user system
+    rlimit memlock 1073741824 1073741824
+    oneshot
+    reboot_on_failure reboot,bpfloader-failed
     override

netd/BpfHandler.cpp

@@ -165,8 +165,35 @@ BpfHandler::BpfHandler()
 BpfHandler::BpfHandler(uint32_t perUidLimit, uint32_t totalLimit)
     : mPerUidStatsEntriesLimit(perUidLimit), mTotalUidStatsEntriesLimit(totalLimit) {}
 
+// copied with minor changes from waitForProgsLoaded()
+// p/m/C's staticlibs/native/bpf_headers/include/bpf/WaitForProgsLoaded.h
+static inline void waitForNetProgsLoaded() {
+    // infinite loop until success with 5/10/20/40/60/60/60... delay
+    for (int delay = 5;; delay *= 2) {
+        if (delay > 60) delay = 60;
+        if (base::WaitForProperty("init.svc.bpfloader", "stopped", std::chrono::seconds(delay))
+            && !access("/sys/fs/bpf/netd_shared", F_OK))
+            return;
+        ALOGW("Waited %ds for init.svc.bpfloader=stopped, still waiting...", delay);
+    }
+}
+
 Status BpfHandler::init(const char* cg2_path) {
     // Make sure BPF programs are loaded before doing anything
+    ALOGI("Waiting for BPF programs");
+
+    if (true || !modules::sdklevel::IsAtLeastV()) {
+        waitForNetProgsLoaded();
+        ALOGI("Networking BPF programs are loaded");
+
+        if (!base::SetProperty("ctl.start", "mdnsd_loadbpf")) {
+            ALOGE("Failed to set property ctl.start=mdnsd_loadbpf, see dmesg for reason.");
+            abort();
+        }
+
+        ALOGI("Waiting for remaining BPF programs");
+    }
+
     android::bpf::waitForProgsLoaded();
     ALOGI("BPF programs are loaded");
 

service/jni/com_android_server_connectivity_ClatCoordinator.cpp

@@ -113,7 +113,12 @@ static void verifyClatPerms() {
     if (!modules::sdklevel::IsAtLeastT()) return;
 
     V("/sys/fs/bpf", S_IFDIR|S_ISVTX|0777, ROOT, ROOT, "fs_bpf", DIR);
-    V("/sys/fs/bpf/net_shared", S_IFDIR|S_ISVTX|0777, ROOT, ROOT, "fs_bpf_net_shared", DIR);
+
+    if (false && modules::sdklevel::IsAtLeastV()) {
+        V("/sys/fs/bpf/net_shared", S_IFDIR|01777, ROOT, ROOT, "fs_bpf_net_shared", DIR);
+    } else {
+        V("/sys/fs/bpf/net_shared", S_IFDIR|01777, SYSTEM, SYSTEM, "fs_bpf_net_shared", DIR);
+    }
 
     // pre-U we do not have selinux privs to getattr on bpf maps/progs
     // so while the below *should* be as listed, we have no way to actually verify