Merge "[Thread] restarts when ot-daemon crashes" into main

thread/service/java/com/android/server/thread/ThreadNetworkControllerService.java

@@ -66,7 +66,6 @@ import android.annotation.RequiresPermission;
 import android.annotation.TargetApi;
 import android.content.Context;
 import android.net.ConnectivityManager;
-import android.net.IpPrefix;
 import android.net.LinkAddress;
 import android.net.LinkProperties;
 import android.net.LocalNetworkConfig;
@@ -79,7 +78,6 @@ import android.net.NetworkCapabilities;
 import android.net.NetworkProvider;
 import android.net.NetworkRequest;
 import android.net.NetworkScore;
-import android.net.RouteInfo;
 import android.net.TestNetworkSpecifier;
 import android.net.thread.ActiveOperationalDataset;
 import android.net.thread.ActiveOperationalDataset.SecurityPolicy;
@@ -151,7 +149,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
     private final ConnectivityManager mConnectivityManager;
     private final TunInterfaceController mTunIfController;
     private final InfraInterfaceController mInfraIfController;
-    private final LinkProperties mLinkProperties = new LinkProperties();
     private final OtDaemonCallbackProxy mOtDaemonCallbackProxy = new OtDaemonCallbackProxy();
 
     // TODO(b/308310823): read supported channel from Thread dameon
@@ -295,12 +292,16 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         return mOtDaemon;
     }
 
-    // TODO(b/309792480): restarts the OT daemon service
     private void onOtDaemonDied() {
-        Log.w(TAG, "OT daemon became dead, clean up...");
+        checkOnHandlerThread();
+        Log.w(TAG, "OT daemon is dead, clean up and restart it...");
+
         OperationReceiverWrapper.onOtDaemonDied();
         mOtDaemonCallbackProxy.onOtDaemonDied();
+        mTunIfController.onOtDaemonDied();
+
         mOtDaemon = null;
+        initializeOtDaemon();
     }
 
     public void initialize() {
@@ -313,8 +314,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
                         throw new IllegalStateException(
                                 "Failed to create Thread tunnel interface", e);
                     }
-                    mLinkProperties.setInterfaceName(TUN_IF_NAME);
-                    mLinkProperties.setMtu(TunInterfaceController.MTU);
                     mConnectivityManager.registerNetworkProvider(mNetworkProvider);
                     requestUpstreamNetwork();
                     requestThreadNetwork();
@@ -465,7 +464,7 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
                 mHandler.getLooper(),
                 TAG,
                 netCaps,
-                mLinkProperties,
+                mTunIfController.getLinkProperties(),
                 newLocalNetworkConfig(),
                 score,
                 new NetworkAgentConfig.Builder().build(),
@@ -496,46 +495,6 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         mNetworkAgent = null;
     }
 
-    private void updateTunInterfaceAddress(LinkAddress linkAddress, boolean isAdded) {
-        try {
-            if (isAdded) {
-                mTunIfController.addAddress(linkAddress);
-            } else {
-                mTunIfController.removeAddress(linkAddress);
-            }
-        } catch (IOException e) {
-            Log.e(
-                    TAG,
-                    String.format(
-                            "Failed to %s Thread tun interface address %s",
-                            (isAdded ? "add" : "remove"), linkAddress),
-                    e);
-        }
-    }
-
-    private void updateNetworkLinkProperties(LinkAddress linkAddress, boolean isAdded) {
-        RouteInfo routeInfo =
-                new RouteInfo(
-                        new IpPrefix(linkAddress.getAddress(), 64),
-                        null,
-                        TUN_IF_NAME,
-                        RouteInfo.RTN_UNICAST,
-                        TunInterfaceController.MTU);
-        if (isAdded) {
-            mLinkProperties.addLinkAddress(linkAddress);
-            mLinkProperties.addRoute(routeInfo);
-        } else {
-            mLinkProperties.removeLinkAddress(linkAddress);
-            mLinkProperties.removeRoute(routeInfo);
-        }
-
-        // The Thread daemon can send link property updates before the networkAgent is
-        // registered
-        if (mNetworkAgent != null) {
-            mNetworkAgent.sendLinkProperties(mLinkProperties);
-        }
-    }
-
     @Override
     public int getThreadVersion() {
         return THREAD_VERSION_1_3;
@@ -872,7 +831,7 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
     private void handleThreadInterfaceStateChanged(boolean isUp) {
         try {
             mTunIfController.setInterfaceUp(isUp);
-            Log.d(TAG, "Thread network interface becomes " + (isUp ? "up" : "down"));
+            Log.i(TAG, "Thread TUN interface becomes " + (isUp ? "up" : "down"));
         } catch (IOException e) {
             Log.e(TAG, "Failed to handle Thread interface state changes", e);
         }
@@ -880,13 +839,13 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
 
     private void handleDeviceRoleChanged(@DeviceRole int deviceRole) {
         if (ThreadNetworkController.isAttached(deviceRole)) {
-            Log.d(TAG, "Attached to the Thread network");
+            Log.i(TAG, "Attached to the Thread network");
 
             // This is an idempotent method which can be called for multiple times when the device
             // is already attached (e.g. going from Child to Router)
             registerThreadNetwork();
         } else {
-            Log.d(TAG, "Detached from the Thread network");
+            Log.i(TAG, "Detached from the Thread network");
 
             // This is an idempotent method which can be called for multiple times when the device
             // is already detached or stopped
@@ -903,10 +862,17 @@ final class ThreadNetworkControllerService extends IThreadNetworkController.Stub
         }
 
         LinkAddress linkAddress = newLinkAddress(addressInfo);
-        Log.d(TAG, (isAdded ? "Adding" : "Removing") + " address " + linkAddress);
+        if (isAdded) {
+            mTunIfController.addAddress(linkAddress);
+        } else {
+            mTunIfController.removeAddress(linkAddress);
+        }
 
-        updateTunInterfaceAddress(linkAddress, isAdded);
+        // The OT daemon can send link property updates before the networkAgent is
-        updateNetworkLinkProperties(linkAddress, isAdded);
+        // registered
+        if (mNetworkAgent != null) {
+            mNetworkAgent.sendLinkProperties(mTunIfController.getLinkProperties());
+        }
     }
 
     private boolean isMulticastForwardingEnabled() {

thread/service/java/com/android/server/thread/TunInterfaceController.java

@@ -17,7 +17,10 @@
 package com.android.server.thread;
 
 import android.annotation.Nullable;
+import android.net.IpPrefix;
 import android.net.LinkAddress;
+import android.net.LinkProperties;
+import android.net.RouteInfo;
 import android.net.util.SocketUtils;
 import android.os.ParcelFileDescriptor;
 import android.os.SystemClock;
@@ -31,6 +34,7 @@ import com.android.net.module.util.netlink.RtNetlinkAddressMessage;
 
 import java.io.FileDescriptor;
 import java.io.IOException;
+import java.io.InterruptedIOException;
 
 /** Controller for virtual/tunnel network interfaces. */
 public class TunInterfaceController {
@@ -43,13 +47,21 @@ public class TunInterfaceController {
     }
 
     private final String mIfName;
+    private final LinkProperties mLinkProperties = new LinkProperties();
     private ParcelFileDescriptor mParcelTunFd;
     private FileDescriptor mNetlinkSocket;
     private static int sNetlinkSeqNo = 0;
 
     /** Creates a new {@link TunInterfaceController} instance for given interface. */
     public TunInterfaceController(String interfaceName) {
-        this.mIfName = interfaceName;
+        mIfName = interfaceName;
+        mLinkProperties.setInterfaceName(mIfName);
+        mLinkProperties.setMtu(MTU);
+    }
+
+    /** Returns link properties of the Thread TUN interface. */
+    public LinkProperties getLinkProperties() {
+        return mLinkProperties;
     }
 
     /**
@@ -87,13 +99,18 @@ public class TunInterfaceController {
 
     /** Sets the interface up or down according to {@code isUp}. */
     public void setInterfaceUp(boolean isUp) throws IOException {
+        if (!isUp) {
+            for (LinkAddress address : mLinkProperties.getAllLinkAddresses()) {
+                removeAddress(address);
+            }
+        }
         nativeSetInterfaceUp(mIfName, isUp);
     }
 
     private native void nativeSetInterfaceUp(String interfaceName, boolean isUp) throws IOException;
 
     /** Adds a new address to the interface. */
-    public void addAddress(LinkAddress address) throws IOException {
+    public void addAddress(LinkAddress address) {
         Log.d(TAG, "Adding address " + address + " with flags: " + address.getFlags());
 
         long validLifetimeSeconds;
@@ -121,7 +138,7 @@ public class TunInterfaceController {
 
         byte[] message =
                 RtNetlinkAddressMessage.newRtmNewAddressMessage(
-                        sNetlinkSeqNo,
+                        sNetlinkSeqNo++,
                         address.getAddress(),
                         (short) address.getPrefixLength(),
                         address.getFlags(),
@@ -131,13 +148,51 @@ public class TunInterfaceController {
                         preferredLifetimeSeconds);
         try {
             Os.write(mNetlinkSocket, message, 0, message.length);
-        } catch (ErrnoException e) {
+        } catch (ErrnoException | InterruptedIOException e) {
-            throw new IOException("Failed to send netlink message", e);
+            Log.e(TAG, "Failed to add address " + address, e);
+            return;
         }
+        mLinkProperties.addLinkAddress(address);
+        mLinkProperties.addRoute(getRouteForAddress(address));
     }
 
     /** Removes an address from the interface. */
-    public void removeAddress(LinkAddress address) throws IOException {
+    public void removeAddress(LinkAddress address) {
-        // TODO(b/263222068): remove address with netlink
+        Log.d(TAG, "Removing address " + address);
+        byte[] message =
+                RtNetlinkAddressMessage.newRtmDelAddressMessage(
+                        sNetlinkSeqNo++,
+                        address.getAddress(),
+                        (short) address.getPrefixLength(),
+                        Os.if_nametoindex(mIfName));
+
+        // Intentionally update the mLinkProperties before send netlink message because the
+        // address is already removed from ot-daemon and apps can't reach to the address even
+        // when the netlink request below fails
+        mLinkProperties.removeLinkAddress(address);
+        mLinkProperties.removeRoute(getRouteForAddress(address));
+        try {
+            Os.write(mNetlinkSocket, message, 0, message.length);
+        } catch (ErrnoException | InterruptedIOException e) {
+            Log.e(TAG, "Failed to remove address " + address, e);
+        }
+    }
+
+    private RouteInfo getRouteForAddress(LinkAddress linkAddress) {
+        return new RouteInfo(
+                new IpPrefix(linkAddress.getAddress(), linkAddress.getPrefixLength()),
+                null,
+                mIfName,
+                RouteInfo.RTN_UNICAST,
+                MTU);
+    }
+
+    /** Called by {@link ThreadNetworkControllerService} to do clean up when ot-daemon is dead. */
+    public void onOtDaemonDied() {
+        try {
+            setInterfaceUp(false);
+        } catch (IOException e) {
+            Log.e(TAG, "Failed to set Thread TUN interface down");
+        }
     }
 }

thread/tests/integration/Android.bp

@@ -24,12 +24,14 @@ java_defaults {
     min_sdk_version: "30",
     static_libs: [
         "androidx.test.rules",
+        "compatibility-device-util-axt",
         "guava",
         "mockito-target-minus-junit4",
         "net-tests-utils",
         "net-utils-device-common",
         "net-utils-device-common-bpf",
         "testables",
+        "truth",
     ],
     libs: [
         "android.test.runner",

thread/tests/integration/AndroidTest.xml

@@ -31,6 +31,8 @@
         <option name="mainline-module-package-name" value="com.google.android.tethering" />
     </object>
 
+    <target_preparer class="com.android.tradefed.targetprep.RootTargetPreparer" />
+
     <!-- Install test -->
     <target_preparer class="com.android.tradefed.targetprep.suite.SuiteApkInstaller">
         <option name="test-file-name" value="ThreadNetworkIntegrationTests.apk" />

thread/tests/integration/src/android/net/thread/BorderRoutingTest.java

@@ -18,14 +18,15 @@ package android.net.thread;
 
 import static android.Manifest.permission.MANAGE_TEST_NETWORKS;
 import static android.Manifest.permission.NETWORK_SETTINGS;
-import static android.net.thread.IntegrationTestUtils.isExpectedIcmpv6Packet;
-import static android.net.thread.IntegrationTestUtils.isSimulatedThreadRadioSupported;
-import static android.net.thread.IntegrationTestUtils.newPacketReader;
-import static android.net.thread.IntegrationTestUtils.readPacketFrom;
-import static android.net.thread.IntegrationTestUtils.waitFor;
-import static android.net.thread.IntegrationTestUtils.waitForStateAnyOf;
 import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_LEADER;
 import static android.net.thread.ThreadNetworkManager.PERMISSION_THREAD_NETWORK_PRIVILEGED;
+import static android.net.thread.utils.IntegrationTestUtils.JOIN_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.isExpectedIcmpv6Packet;
+import static android.net.thread.utils.IntegrationTestUtils.isSimulatedThreadRadioSupported;
+import static android.net.thread.utils.IntegrationTestUtils.newPacketReader;
+import static android.net.thread.utils.IntegrationTestUtils.readPacketFrom;
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
+import static android.net.thread.utils.IntegrationTestUtils.waitForStateAnyOf;
 
 import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ECHO_REPLY_TYPE;
 import static com.android.testutils.TestNetworkTrackerKt.initTestNetwork;
@@ -41,6 +42,8 @@ import static org.junit.Assume.assumeTrue;
 import android.content.Context;
 import android.net.LinkProperties;
 import android.net.MacAddress;
+import android.net.thread.utils.FullThreadDevice;
+import android.net.thread.utils.InfraNetworkDevice;
 import android.os.Handler;
 import android.os.HandlerThread;
 
@@ -57,6 +60,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import java.net.Inet6Address;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -155,14 +159,14 @@ public class BorderRoutingTest {
         runAsShell(
                 PERMISSION_THREAD_NETWORK_PRIVILEGED,
                 () -> mController.join(DEFAULT_DATASET, directExecutor(), result -> {}));
-        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), 30 /* timeoutSeconds */);
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), JOIN_TIMEOUT);
 
         // Creates a Full Thread Device (FTD) and lets it join the network.
         FullThreadDevice ftd = new FullThreadDevice(5 /* node ID */);
         ftd.factoryReset();
         ftd.joinNetwork(DEFAULT_DATASET);
-        ftd.waitForStateAnyOf(List.of("router", "child"), 10 /* timeoutSeconds */);
+        ftd.waitForStateAnyOf(List.of("router", "child"), JOIN_TIMEOUT);
-        waitFor(() -> ftd.getOmrAddress() != null, 60 /* timeoutSeconds */);
+        waitFor(() -> ftd.getOmrAddress() != null, Duration.ofSeconds(60));
         Inet6Address ftdOmr = ftd.getOmrAddress();
         assertNotNull(ftdOmr);
 
@@ -171,7 +175,7 @@ public class BorderRoutingTest {
                 newPacketReader(mInfraNetworkTracker.getTestIface(), mHandler);
         InfraNetworkDevice infraDevice =
                 new InfraNetworkDevice(MacAddress.fromString("1:2:3:4:5:6"), infraNetworkReader);
-        infraDevice.runSlaac(60 /* timeoutSeconds */);
+        infraDevice.runSlaac(Duration.ofSeconds(60));
         assertNotNull(infraDevice.ipv6Addr);
 
         // Infra device sends an echo request to FTD's OMR.

thread/tests/integration/src/android/net/thread/ThreadIntegrationTest.java

@@ -0,0 +1,198 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net.thread;
+
+import static android.Manifest.permission.NETWORK_SETTINGS;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_DETACHED;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_LEADER;
+import static android.net.thread.ThreadNetworkController.DEVICE_ROLE_STOPPED;
+import static android.net.thread.ThreadNetworkManager.PERMISSION_THREAD_NETWORK_PRIVILEGED;
+import static android.net.thread.utils.IntegrationTestUtils.CALLBACK_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.LEAVE_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.RESTART_JOIN_TIMEOUT;
+import static android.net.thread.utils.IntegrationTestUtils.waitForStateAnyOf;
+
+import static com.android.compatibility.common.util.SystemUtil.runShellCommand;
+import static com.android.testutils.TestPermissionUtil.runAsShell;
+
+import static com.google.common.io.BaseEncoding.base16;
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+
+import static org.junit.Assume.assumeNotNull;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import android.annotation.Nullable;
+import android.content.Context;
+import android.net.thread.ThreadNetworkController.StateCallback;
+import android.net.thread.utils.OtDaemonController;
+import android.os.SystemClock;
+
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.filters.LargeTest;
+import androidx.test.runner.AndroidJUnit4;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.net.Inet6Address;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+/** Tests for E2E Android Thread integration with ot-daemon, ConnectivityService, etc.. */
+@LargeTest
+@RunWith(AndroidJUnit4.class)
+public class ThreadIntegrationTest {
+    private final Context mContext = ApplicationProvider.getApplicationContext();
+    private ThreadNetworkController mController;
+    private OtDaemonController mOtCtl;
+
+    // A valid Thread Active Operational Dataset generated from OpenThread CLI "dataset init new".
+    private static final byte[] DEFAULT_DATASET_TLVS =
+            base16().decode(
+                            "0E080000000000010000000300001335060004001FFFE002"
+                                    + "08ACC214689BC40BDF0708FD64DB1225F47E0B0510F26B31"
+                                    + "53760F519A63BAFDDFFC80D2AF030F4F70656E5468726561"
+                                    + "642D643961300102D9A00410A245479C836D551B9CA557F7"
+                                    + "B9D351B40C0402A0FFF8");
+    private static final ActiveOperationalDataset DEFAULT_DATASET =
+            ActiveOperationalDataset.fromThreadTlvs(DEFAULT_DATASET_TLVS);
+
+    @Before
+    public void setUp() throws Exception {
+        final ThreadNetworkManager manager = mContext.getSystemService(ThreadNetworkManager.class);
+        if (manager != null) {
+            mController = manager.getAllThreadNetworkControllers().get(0);
+        }
+
+        // Run the tests on only devices where the Thread feature is available
+        assumeNotNull(mController);
+
+        mOtCtl = new OtDaemonController();
+        leaveAndWait(mController);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        if (mController == null) {
+            return;
+        }
+
+        setTestUpStreamNetworkAndWait(mController, null);
+        leaveAndWait(mController);
+    }
+
+    @Test
+    public void otDaemonRestart_notJoinedAndStopped_deviceRoleIsStopped() throws Exception {
+        leaveAndWait(mController);
+
+        runShellCommand("stop ot-daemon");
+        // TODO(b/323331973): the sleep is needed to workaround the race conditions
+        SystemClock.sleep(200);
+
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_STOPPED), CALLBACK_TIMEOUT);
+    }
+
+    @Test
+    public void otDaemonRestart_JoinedNetworkAndStopped_autoRejoined() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        runShellCommand("stop ot-daemon");
+
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_DETACHED), CALLBACK_TIMEOUT);
+        waitForStateAnyOf(mController, List.of(DEVICE_ROLE_LEADER), RESTART_JOIN_TIMEOUT);
+    }
+
+    @Test
+    public void otDaemonFactoryReset_deviceRoleIsStopped() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        mOtCtl.factoryReset();
+
+        assertThat(getDeviceRole(mController)).isEqualTo(DEVICE_ROLE_STOPPED);
+    }
+
+    @Test
+    public void otDaemonFactoryReset_addressesRemoved() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        mOtCtl.factoryReset();
+        String ifconfig = runShellCommand("ifconfig thread-wpan");
+
+        assertThat(ifconfig).doesNotContain("inet6 addr");
+    }
+
+    @Test
+    public void tunInterface_joinedNetwork_otAddressesAddedToTunInterface() throws Exception {
+        joinAndWait(mController, DEFAULT_DATASET);
+
+        String ifconfig = runShellCommand("ifconfig thread-wpan");
+        List<Inet6Address> otAddresses = mOtCtl.getAddresses();
+        assertThat(otAddresses).isNotEmpty();
+        for (Inet6Address otAddress : otAddresses) {
+            assertThat(ifconfig).contains(otAddress.getHostAddress());
+        }
+    }
+
+    // TODO (b/323300829): add more tests for integration with linux platform and
+    // ConnectivityService
+
+    private static int getDeviceRole(ThreadNetworkController controller) throws Exception {
+        CompletableFuture<Integer> future = new CompletableFuture<>();
+        StateCallback callback = future::complete;
+        controller.registerStateCallback(directExecutor(), callback);
+        try {
+            return future.get(CALLBACK_TIMEOUT.toMillis(), MILLISECONDS);
+        } finally {
+            controller.unregisterStateCallback(callback);
+        }
+    }
+
+    private static void joinAndWait(
+            ThreadNetworkController controller, ActiveOperationalDataset activeDataset)
+            throws Exception {
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                () -> controller.join(activeDataset, directExecutor(), result -> {}));
+        waitForStateAnyOf(controller, List.of(DEVICE_ROLE_LEADER), RESTART_JOIN_TIMEOUT);
+    }
+
+    private static void leaveAndWait(ThreadNetworkController controller) throws Exception {
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                () -> controller.leave(directExecutor(), future::complete));
+        future.get(LEAVE_TIMEOUT.toMillis(), MILLISECONDS);
+    }
+
+    private static void setTestUpStreamNetworkAndWait(
+            ThreadNetworkController controller, @Nullable String networkInterfaceName)
+            throws Exception {
+        CompletableFuture<Void> future = new CompletableFuture<>();
+        runAsShell(
+                PERMISSION_THREAD_NETWORK_PRIVILEGED,
+                NETWORK_SETTINGS,
+                () -> {
+                    controller.setTestNetworkAsUpstream(
+                            networkInterfaceName, directExecutor(), future::complete);
+                });
+        future.get(CALLBACK_TIMEOUT.toMillis(), MILLISECONDS);
+    }
+}

thread/tests/integration/src/android/net/thread/utils/FullThreadDevice.java

@@ -0,0 +1,182 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.net.thread.utils;
+
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
+
+import static com.google.common.io.BaseEncoding.base16;
+
+import static org.junit.Assert.fail;
+
+import android.net.InetAddresses;
+import android.net.IpPrefix;
+import android.net.thread.ActiveOperationalDataset;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.net.Inet6Address;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * A class that launches and controls a simulation Full Thread Device (FTD).
+ *
+ * <p>This class launches an `ot-cli-ftd` process and communicates with it via command line input
+ * and output. See <a
+ * href="https://github.com/openthread/openthread/blob/main/src/cli/README.md">this page</a> for
+ * available commands.
+ */
+public final class FullThreadDevice {
+    private final Process mProcess;
+    private final BufferedReader mReader;
+    private final BufferedWriter mWriter;
+
+    private ActiveOperationalDataset mActiveOperationalDataset;
+
+    /**
+     * Constructs a {@link FullThreadDevice} for the given node ID.
+     *
+     * <p>It launches an `ot-cli-ftd` process using the given node ID. The node ID is an integer in
+     * range [1, OPENTHREAD_SIMULATION_MAX_NETWORK_SIZE]. `OPENTHREAD_SIMULATION_MAX_NETWORK_SIZE`
+     * is defined in `external/openthread/examples/platforms/simulation/platform-config.h`.
+     *
+     * @param nodeId the node ID for the simulation Full Thread Device.
+     * @throws IllegalStateException the node ID is already occupied by another simulation Thread
+     *     device.
+     */
+    public FullThreadDevice(int nodeId) {
+        try {
+            mProcess = Runtime.getRuntime().exec("/system/bin/ot-cli-ftd " + nodeId);
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to start ot-cli-ftd (id=" + nodeId + ")", e);
+        }
+        mReader = new BufferedReader(new InputStreamReader(mProcess.getInputStream()));
+        mWriter = new BufferedWriter(new OutputStreamWriter(mProcess.getOutputStream()));
+        mActiveOperationalDataset = null;
+    }
+
+    /**
+     * Returns an OMR (Off-Mesh-Routable) address on this device if any.
+     *
+     * <p>This methods goes through all unicast addresses on the device and returns the first
+     * address which is neither link-local nor mesh-local.
+     */
+    public Inet6Address getOmrAddress() {
+        List<String> addresses = executeCommand("ipaddr");
+        IpPrefix meshLocalPrefix = mActiveOperationalDataset.getMeshLocalPrefix();
+        for (String address : addresses) {
+            if (address.startsWith("fe80:")) {
+                continue;
+            }
+            Inet6Address addr = (Inet6Address) InetAddresses.parseNumericAddress(address);
+            if (!meshLocalPrefix.contains(addr)) {
+                return addr;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Joins the Thread network using the given {@link ActiveOperationalDataset}.
+     *
+     * @param dataset the Active Operational Dataset
+     */
+    public void joinNetwork(ActiveOperationalDataset dataset) {
+        mActiveOperationalDataset = dataset;
+        executeCommand("dataset set active " + base16().lowerCase().encode(dataset.toThreadTlvs()));
+        executeCommand("ifconfig up");
+        executeCommand("thread start");
+    }
+
+    /** Stops the Thread network radio. */
+    public void stopThreadRadio() {
+        executeCommand("thread stop");
+        executeCommand("ifconfig down");
+    }
+
+    /**
+     * Waits for the Thread device to enter the any state of the given {@link List<String>}.
+     *
+     * @param states the list of states to wait for. Valid states are "disabled", "detached",
+     *     "child", "router" and "leader".
+     * @param timeout the time to wait for the expected state before throwing
+     */
+    public void waitForStateAnyOf(List<String> states, Duration timeout) throws TimeoutException {
+        waitFor(() -> states.contains(getState()), timeout);
+    }
+
+    /**
+     * Gets the state of the Thread device.
+     *
+     * @return a string representing the state.
+     */
+    public String getState() {
+        return executeCommand("state").get(0);
+    }
+
+    /** Runs the "factoryreset" command on the device. */
+    public void factoryReset() {
+        try {
+            mWriter.write("factoryreset\n");
+            mWriter.flush();
+            // fill the input buffer to avoid truncating next command
+            for (int i = 0; i < 1000; ++i) {
+                mWriter.write("\n");
+            }
+            mWriter.flush();
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to run factoryreset on ot-cli-ftd", e);
+        }
+    }
+
+    private List<String> executeCommand(String command) {
+        try {
+            mWriter.write(command + "\n");
+            mWriter.flush();
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                    "Failed to write the command " + command + " to ot-cli-ftd", e);
+        }
+        try {
+            return readUntilDone();
+        } catch (IOException e) {
+            throw new IllegalStateException(
+                    "Failed to read the ot-cli-ftd output of command: " + command, e);
+        }
+    }
+
+    private List<String> readUntilDone() throws IOException {
+        ArrayList<String> result = new ArrayList<>();
+        String line;
+        while ((line = mReader.readLine()) != null) {
+            if (line.equals("Done")) {
+                break;
+            }
+            if (line.startsWith("Error:")) {
+                fail("ot-cli-ftd reported an error: " + line);
+            }
+            if (!line.startsWith("> ")) {
+                result.add(line);
+            }
+        }
+        return result;
+    }
+}

thread/tests/integration/src/android/net/thread/utils/InfraNetworkDevice.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.net.thread.utils;
+
+import static android.net.thread.utils.IntegrationTestUtils.getRaPios;
+import static android.net.thread.utils.IntegrationTestUtils.readPacketFrom;
+import static android.net.thread.utils.IntegrationTestUtils.waitFor;
+
+import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ND_OPTION_SLLA;
+import static com.android.net.module.util.NetworkStackConstants.IPV6_ADDR_ALL_ROUTERS_MULTICAST;
+
+import android.net.InetAddresses;
+import android.net.MacAddress;
+
+import com.android.net.module.util.Ipv6Utils;
+import com.android.net.module.util.structs.LlaOption;
+import com.android.net.module.util.structs.PrefixInformationOption;
+import com.android.testutils.TapPacketReader;
+
+import java.io.IOException;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * A class that simulates a device on the infrastructure network.
+ *
+ * <p>This class directly interacts with the TUN interface of the test network to pretend there's a
+ * device on the infrastructure network.
+ */
+public final class InfraNetworkDevice {
+    // The MAC address of this device.
+    public final MacAddress macAddr;
+    // The packet reader of the TUN interface of the test network.
+    public final TapPacketReader packetReader;
+    // The IPv6 address generated by SLAAC for the device.
+    public Inet6Address ipv6Addr;
+
+    /**
+     * Constructs an InfraNetworkDevice with the given {@link MAC address} and {@link
+     * TapPacketReader}.
+     *
+     * @param macAddr the MAC address of the device
+     * @param packetReader the packet reader of the TUN interface of the test network.
+     */
+    public InfraNetworkDevice(MacAddress macAddr, TapPacketReader packetReader) {
+        this.macAddr = macAddr;
+        this.packetReader = packetReader;
+    }
+
+    /**
+     * Sends an ICMPv6 echo request message to the given {@link Inet6Address}.
+     *
+     * @param dstAddr the destination address of the packet.
+     * @throws IOException when it fails to send the packet.
+     */
+    public void sendEchoRequest(Inet6Address dstAddr) throws IOException {
+        ByteBuffer icmp6Packet = Ipv6Utils.buildEchoRequestPacket(ipv6Addr, dstAddr);
+        packetReader.sendResponse(icmp6Packet);
+    }
+
+    /**
+     * Sends an ICMPv6 Router Solicitation (RS) message to all routers on the network.
+     *
+     * @throws IOException when it fails to send the packet.
+     */
+    public void sendRsPacket() throws IOException {
+        ByteBuffer slla = LlaOption.build((byte) ICMPV6_ND_OPTION_SLLA, macAddr);
+        ByteBuffer rs =
+                Ipv6Utils.buildRsPacket(
+                        (Inet6Address) InetAddresses.parseNumericAddress("fe80::1"),
+                        IPV6_ADDR_ALL_ROUTERS_MULTICAST,
+                        slla);
+        packetReader.sendResponse(rs);
+    }
+
+    /**
+     * Runs SLAAC to generate an IPv6 address for the device.
+     *
+     * <p>The devices sends an RS message, processes the received RA messages and generates an IPv6
+     * address if there's any available Prefix Information Option (PIO). For now it only generates
+     * one address in total and doesn't track the expiration.
+     *
+     * @param timeoutSeconds the number of seconds to wait for.
+     * @throws TimeoutException when the device fails to generate a SLAAC address in given timeout.
+     */
+    public void runSlaac(Duration timeout) throws TimeoutException {
+        waitFor(() -> (ipv6Addr = runSlaac()) != null, timeout);
+    }
+
+    private Inet6Address runSlaac() {
+        try {
+            sendRsPacket();
+
+            final byte[] raPacket = readPacketFrom(packetReader, p -> !getRaPios(p).isEmpty());
+
+            final List<PrefixInformationOption> options = getRaPios(raPacket);
+
+            for (PrefixInformationOption pio : options) {
+                if (pio.validLifetime > 0 && pio.preferredLifetime > 0) {
+                    final byte[] addressBytes = pio.prefix;
+                    addressBytes[addressBytes.length - 1] = (byte) (new Random()).nextInt();
+                    addressBytes[addressBytes.length - 2] = (byte) (new Random()).nextInt();
+                    return (Inet6Address) InetAddress.getByAddress(addressBytes);
+                }
+            }
+        } catch (IOException e) {
+            throw new IllegalStateException("Failed to generate an address by SLAAC", e);
+        }
+        return null;
+    }
+}

thread/tests/integration/src/android/net/thread/utils/IntegrationTestUtils.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.net.thread.utils;
+
+import static android.system.OsConstants.IPPROTO_ICMPV6;
+
+import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ND_OPTION_PIO;
+import static com.android.net.module.util.NetworkStackConstants.ICMPV6_ROUTER_ADVERTISEMENT;
+
+import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
+
+import android.net.TestNetworkInterface;
+import android.net.thread.ThreadNetworkController;
+import android.os.Handler;
+import android.os.SystemClock;
+import android.os.SystemProperties;
+
+import com.android.net.module.util.Struct;
+import com.android.net.module.util.structs.Icmpv6Header;
+import com.android.net.module.util.structs.Ipv6Header;
+import com.android.net.module.util.structs.PrefixInformationOption;
+import com.android.net.module.util.structs.RaHeader;
+import com.android.testutils.HandlerUtils;
+import com.android.testutils.TapPacketReader;
+
+import com.google.common.util.concurrent.SettableFuture;
+
+import java.io.FileDescriptor;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+/** Static utility methods relating to Thread integration tests. */
+public final class IntegrationTestUtils {
+    // The timeout of join() after restarting ot-daemon. The device needs to send 6 Link Request
+    // every 5 seconds, followed by 4 Parent Request every second. So this value needs to be 40
+    // seconds to be safe
+    public static final Duration RESTART_JOIN_TIMEOUT = Duration.ofSeconds(40);
+    public static final Duration JOIN_TIMEOUT = Duration.ofSeconds(30);
+    public static final Duration LEAVE_TIMEOUT = Duration.ofSeconds(2);
+    public static final Duration CALLBACK_TIMEOUT = Duration.ofSeconds(1);
+
+    private IntegrationTestUtils() {}
+
+    /** Returns whether the device supports simulated Thread radio. */
+    public static boolean isSimulatedThreadRadioSupported() {
+        // The integration test uses SIMULATION Thread radio so that it only supports CuttleFish.
+        return SystemProperties.get("ro.product.model").startsWith("Cuttlefish");
+    }
+
+    /**
+     * Waits for the given {@link Supplier} to be true until given timeout.
+     *
+     * @param condition the condition to check
+     * @param timeout the time to wait for the condition before throwing
+     * @throws TimeoutException if the condition is still not met when the timeout expires
+     */
+    public static void waitFor(Supplier<Boolean> condition, Duration timeout)
+            throws TimeoutException {
+        final long intervalMills = 1000;
+        final long timeoutMills = timeout.toMillis();
+
+        for (long i = 0; i < timeoutMills; i += intervalMills) {
+            if (condition.get()) {
+                return;
+            }
+            SystemClock.sleep(intervalMills);
+        }
+        if (condition.get()) {
+            return;
+        }
+        throw new TimeoutException("The condition failed to become true in " + timeout);
+    }
+
+    /**
+     * Creates a {@link TapPacketReader} given the {@link TestNetworkInterface} and {@link Handler}.
+     *
+     * @param testNetworkInterface the TUN interface of the test network
+     * @param handler the handler to process the packets
+     * @return the {@link TapPacketReader}
+     */
+    public static TapPacketReader newPacketReader(
+            TestNetworkInterface testNetworkInterface, Handler handler) {
+        FileDescriptor fd = testNetworkInterface.getFileDescriptor().getFileDescriptor();
+        final TapPacketReader reader =
+                new TapPacketReader(handler, fd, testNetworkInterface.getMtu());
+        handler.post(() -> reader.start());
+        HandlerUtils.waitForIdle(handler, 5000 /* timeout in milliseconds */);
+        return reader;
+    }
+
+    /**
+     * Waits for the Thread module to enter any state of the given {@code deviceRoles}.
+     *
+     * @param controller the {@link ThreadNetworkController}
+     * @param deviceRoles the desired device roles. See also {@link
+     *     ThreadNetworkController.DeviceRole}
+     * @param timeout the time to wait for the expected state before throwing
+     * @return the {@link ThreadNetworkController.DeviceRole} after waiting
+     * @throws TimeoutException if the device hasn't become any of expected roles until the timeout
+     *     expires
+     */
+    public static int waitForStateAnyOf(
+            ThreadNetworkController controller, List<Integer> deviceRoles, Duration timeout)
+            throws TimeoutException {
+        SettableFuture<Integer> future = SettableFuture.create();
+        ThreadNetworkController.StateCallback callback =
+                newRole -> {
+                    if (deviceRoles.contains(newRole)) {
+                        future.set(newRole);
+                    }
+                };
+        controller.registerStateCallback(directExecutor(), callback);
+        try {
+            return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS);
+        } catch (InterruptedException | ExecutionException e) {
+            throw new TimeoutException(
+                    String.format(
+                            "The device didn't become an expected role in %s: %s",
+                            timeout, e.getMessage()));
+        } finally {
+            controller.unregisterStateCallback(callback);
+        }
+    }
+
+    /**
+     * Reads a packet from a given {@link TapPacketReader} that satisfies the {@code filter}.
+     *
+     * @param packetReader a TUN packet reader
+     * @param filter the filter to be applied on the packet
+     * @return the first IPv6 packet that satisfies the {@code filter}. If it has waited for more
+     *     than 3000ms to read the next packet, the method will return null
+     */
+    public static byte[] readPacketFrom(TapPacketReader packetReader, Predicate<byte[]> filter) {
+        byte[] packet;
+        while ((packet = packetReader.poll(3000 /* timeoutMs */)) != null) {
+            if (filter.test(packet)) return packet;
+        }
+        return null;
+    }
+
+    /** Returns {@code true} if {@code packet} is an ICMPv6 packet of given {@code type}. */
+    public static boolean isExpectedIcmpv6Packet(byte[] packet, int type) {
+        if (packet == null) {
+            return false;
+        }
+        ByteBuffer buf = ByteBuffer.wrap(packet);
+        try {
+            if (Struct.parse(Ipv6Header.class, buf).nextHeader != (byte) IPPROTO_ICMPV6) {
+                return false;
+            }
+            return Struct.parse(Icmpv6Header.class, buf).type == (short) type;
+        } catch (IllegalArgumentException ignored) {
+            // It's fine that the passed in packet is malformed because it's could be sent
+            // by anybody.
+        }
+        return false;
+    }
+
+    /** Returns the Prefix Information Options (PIO) extracted from an ICMPv6 RA message. */
+    public static List<PrefixInformationOption> getRaPios(byte[] raMsg) {
+        final ArrayList<PrefixInformationOption> pioList = new ArrayList<>();
+
+        if (raMsg == null) {
+            return pioList;
+        }
+
+        final ByteBuffer buf = ByteBuffer.wrap(raMsg);
+        final Ipv6Header ipv6Header = Struct.parse(Ipv6Header.class, buf);
+        if (ipv6Header.nextHeader != (byte) IPPROTO_ICMPV6) {
+            return pioList;
+        }
+
+        final Icmpv6Header icmpv6Header = Struct.parse(Icmpv6Header.class, buf);
+        if (icmpv6Header.type != (short) ICMPV6_ROUTER_ADVERTISEMENT) {
+            return pioList;
+        }
+
+        Struct.parse(RaHeader.class, buf);
+        while (buf.position() < raMsg.length) {
+            final int currentPos = buf.position();
+            final int type = Byte.toUnsignedInt(buf.get());
+            final int length = Byte.toUnsignedInt(buf.get());
+            if (type == ICMPV6_ND_OPTION_PIO) {
+                final ByteBuffer pioBuf =
+                        ByteBuffer.wrap(
+                                buf.array(),
+                                currentPos,
+                                Struct.getSize(PrefixInformationOption.class));
+                final PrefixInformationOption pio =
+                        Struct.parse(PrefixInformationOption.class, pioBuf);
+                pioList.add(pio);
+
+                // Move ByteBuffer position to the next option.
+                buf.position(currentPos + Struct.getSize(PrefixInformationOption.class));
+            } else {
+                // The length is in units of 8 octets.
+                buf.position(currentPos + (length * 8));
+            }
+        }
+        return pioList;
+    }
+}

thread/tests/integration/src/android/net/thread/utils/OtDaemonController.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.net.thread.utils;
+
+import android.net.InetAddresses;
+import android.os.SystemClock;
+
+import com.android.compatibility.common.util.SystemUtil;
+
+import java.net.Inet6Address;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Wrapper of the "/system/bin/ot-ctl" which can be used to send CLI commands to ot-daemon to
+ * control its behavior.
+ *
+ * <p>Note that this class takes root privileged to run.
+ */
+public final class OtDaemonController {
+    private static final String OT_CTL = "/system/bin/ot-ctl";
+
+    /**
+     * Factory resets ot-daemon.
+     *
+     * <p>This will erase all persistent data written into apexdata/com.android.apex/ot-daemon and
+     * restart the ot-daemon service.
+     */
+    public void factoryReset() {
+        executeCommand("factoryreset");
+
+        // TODO(b/323164524): ot-ctl is a separate process so that the tests can't depend on the
+        // time sequence. Here needs to wait for system server to receive the ot-daemon death
+        // signal and take actions.
+        // A proper fix is to replace "ot-ctl" with "cmd thread_network ot-ctl" which is
+        // synchronized with the system server
+        SystemClock.sleep(500);
+    }
+
+    /** Returns the list of IPv6 addresses on ot-daemon. */
+    public List<Inet6Address> getAddresses() {
+        String output = executeCommand("ipaddr");
+        return Arrays.asList(output.split("\n")).stream()
+                .map(String::trim)
+                .filter(str -> !str.equals("Done"))
+                .map(addr -> InetAddresses.parseNumericAddress(addr))
+                .map(inetAddr -> (Inet6Address) inetAddr)
+                .toList();
+    }
+
+    public String executeCommand(String cmd) {
+        return SystemUtil.runShellCommand(OT_CTL + " " + cmd);
+    }
+}