Google celebrates Women's History Month. See how.

Contributing Tests to CTF

This guide will walk you through the process of writing a CTF test for a FIDL API.

To test a FIDL API, we write a test that uses the API's client bindings in the SDK to interact with the FIDL service. For this guide we'll be testing an example FIDL service fuchsia.examples.Echo from the library fuchsia.examples. Throughout this guide, you can replace the library and service with your own values to match your use case.

The test will consist of two components: The first is a test driver binary which implements the core test logic, and is the part of your test that will be released in CTF. The second is a test realm which provides the capabilities and dependencies that we want to test, is always built from the sources at HEAD, and is not released as part of CTF.

Requirements

  • Tests must be written in C, C++, or Rust.
  • Tests must only depend on APIs, ABIs, and tools in partner facing SDKs.

If you are writing a CTF test for a FIDL service whose fidl target is not in the partner SDK category, please see this section.

Concepts

Steps

1. Setup

First create a directory for the test. The directory name should match the name of the FIDL library. You can copy these commands to generate some scaffolding:

C/C++

mkdir sdk/ctf/tests/fidl/fuchsia.examples
mkdir sdk/ctf/tests/fidl/fuchsia.examples/meta/
touch sdk/ctf/tests/fidl/fuchsia.examples/meta/fuchsia.examples.echo_test.cml
touch sdk/ctf/tests/fidl/fuchsia.examples/BUILD.gn
touch sdk/ctf/tests/fidl/fuchsia.examples/main.cc

Rust

mkdir sdk/ctf/tests/fidl/fuchsia.examples
mkdir sdk/ctf/tests/fidl/fuchsia.examples/meta/
# Rust tests need an additional component to offer the subpackaged test runner.
touch sdk/ctf/tests/fidl/fuchsia.examples/meta/fuchsia.examples.echo_test_root.cml
touch sdk/ctf/tests/fidl/fuchsia.examples/meta/fuchsia.examples.echo_test.cml
touch sdk/ctf/tests/fidl/fuchsia.examples/BUILD.gn
touch sdk/ctf/tests/fidl/fuchsia.examples/main.rs

2. Create the test realm

The test realm is a component whose sole purpose is to expose the FIDL API that we want to test. The test realm component is always built from the HEAD of the current Fuchsia branch. This is what makes the CTF test a compatibility test: The test and the FIDL capability it's testing are built at different versions.

For convenience, the test realm component can be defined anywhere in the source tree but we prefer if all realms are defined in //sdk/ctf/test_realm/BUILD.gn.

To create the realm, add contents like the following to //sdk/ctf/test_realm/BUILD.gn:

ctf_test_realm("fuchsia.example_test_realm") {
  manifest = "meta/$target_name.cml"
  deps = [ "//examples/fidl/cpp/server:echo-server" ]
}

Then create the test realm's component manifest at //sdk/ctf/test_realm/meta/fuchsia.examples_test_realm.cml:

{
    include: [ "syslog/client.shard.cml" ],
    children: [
        {
            name: "echo_server",
            url: "#meta/echo_server.cm",
        },
    ],
    expose: [
        {
            protocol: "fuchsia.examples.Echo",
            from: "#echo_server",
        },
    ],
}

Finally, add the realm's label to the list in //sdk/ctf/build/ctf_test_realms.gni. This will cause the build to include the test realm as a subpackage of your test. We'll explain this in more detail, later.

3. Write the BUILD.gn file

Add contents like the following to //sdk/ctf/tests/fidl/fuchsia.examples/BUILD.gn to define an executable, test driver component, and package for your test. Be sure to add the :tests target as a dependency of //sdk/ctf/tests/fidl:tests in order to include the test in the build graph for CI and CQ builds.

C/C++


import("//sdk/ctf/build/ctf.gni")

group("tests") {
  testonly = true
  deps = [ ":fuchsia.examples.echo_test" ]
}

ctf_executable("test_main") {
  output_name = "fuchsia.examples.echo_test_main"
  sources = [ "main.cc" ]
  deps = [
    "//examples/fidl/fuchsia.examples:fuchsia.examples_cpp",
    "//sdk/lib/component/incoming/cpp",
    "//sdk/lib/syslog/cpp:cpp-macros",
    "//zircon/system/ulib/zxtest",
  ]
  testonly = true
}

ctf_fuchsia_component("fuchsia.examples.echo_test_component") {
  component_name = "fuchsia.examples.echo_test"
  manifest = "meta/fuchsia.examples.echo_test.cml"
  deps = [ ":test_main" ]
  testonly = true
}

ctf_fuchsia_test_package("fuchsia.examples.echo_test") {
  enable_ctf_test_realms = true
  package_name = "fuchsia.examples.echo_test"
  test_components = [ ":fuchsia.examples.echo_test_component" ]
}

Rust


import("//sdk/ctf/build/ctf.gni")

group("tests") {
  testonly = true
  deps = [ ":fuchsia.examples.echo_rust_test" ]
}

ctf_rustc_test("test_main") {
  edition = "2021"
  output_name = "fuchsia_examples_echo_test_rust_main"
  source_root = "main.rs"
  sources = [ source_root ]
  deps = [
    "//examples/fidl/fuchsia.examples:fuchsia.examples_rust",
    "//src/lib/fuchsia",
    "//src/lib/fuchsia-component",
    "//src/lib/zircon/rust:fuchsia-zircon",
    "//third_party/rust_crates:anyhow",
  ]
}

ctf_fuchsia_component("fuchsia.examples.echo_test_component") {
  component_name = "fuchsia.examples.echo_test"
  manifest = "meta/fuchsia.examples.echo_test.cml"
  deps = [ ":test_main" ]
  testonly = true
}

ctf_fuchsia_component("fuchsia.examples.echo_test_root") {
  component_name = "fuchsia.examples.echo_test_root"
  manifest = "meta/fuchsia.examples.echo_test_root.cml"
}

ctf_fuchsia_test_package("fuchsia.examples.echo_rust_test") {
  enable_ctf_test_realms = true
  package_name = "fuchsia.examples.echo_rust_test"
  test_components = [ ":fuchsia.examples.echo_test_root" ]
  subpackages = RUST_SUBPACKAGES
  deps = [ ":fuchsia.examples.echo_test_component" ]
}

4. Implement the test driver

This component implements the core logic for your test.

First we need to create the component manifest. Add contents like the following to fuchsia.examples.echo_test.cml:

C/C++

{
    include: [
        "//sdk/ctf/meta/test.shard.cml",
        "sys/testing/elf_test_runner.shard.cml",
        "syslog/client.shard.cml",
    ],
    program: {
        binary: "bin/fuchsia.examples.echo_test_main",
    },
    children: [
        {
            name: "test_realm",
            url: "fuchsia.example_test_realm#meta/default.cm",
        },
    ],
    use: [
        {
            protocol: "fuchsia.examples.Echo",
            from: "#test_realm",
        },
    ],
}

Rust

This test root component includes //sdk/ctf/meta/rust.shard.cml, which defines the rust test runner as a subpackage. The echo_test component must be started in the subpackaged-runner-env.

// echo_test_root.cml

{
    include: [
        "//sdk/ctf/meta/rust.shard.cml",
        "//sdk/ctf/meta/test.shard.cml",
        "syslog/client.shard.cml",
    ],
    children: [
        {
            name: "echo_test",
            url: "#meta/fuchsia.examples.echo_test.cm",
            environment: "#subpackaged-runner-env",
        },
        {
            name: "test_realm",
            url: "fuchsia.example_test_realm#meta/default.cm",
        },
    ],
    offer: [
        {
            protocol: "fuchsia.examples.Echo",
            from: "#test_realm",
            to: "#echo_test",
        },
    ],
    expose: [
        {
            protocol: "fuchsia.test.Suite",
            from: "#echo_test",
        },
    ],
}

The test component. The test realm is offered from the test root rather than the test itself.

// echo_test.cml

{
    include: [
        "//src/sys/test_runners/rust/default.shard.cml",
        "syslog/client.shard.cml",
    ],
    program: {
        binary: "bin/fuchsia_examples_echo_test_rust_main",
    },
    use: [
        { protocol: "fuchsia.examples.Echo" },
    ],
}

The package URL fuchsia.examples_test_realm#meta/default.cm loads the test realm we created earlier from a subpackage. Every time this test runs it receives a new version of this subpackage which is built from the current commit, regardless of whether the test itself is built from the current commit or obtained as a prebuilt.

Next we need to implement the executable. Add contents like the following to the test's source file:

C/C++

#include <fidl/fuchsia.examples/cpp/fidl.h>
#include <lib/component/incoming/cpp/protocol.h>
#include <lib/syslog/cpp/macros.h>

#include <string>

#include <zxtest/zxtest.h>

class FuchsiaExamplesTest : public zxtest::Test {
 public:
  ~FuchsiaExamplesTest() override = default;
};

TEST(FuchsiaExamplesTest, Echo) {
  zx::result client_end = component::Connect<fuchsia_examples::Echo>();

  EXPECT_TRUE(client_end.is_ok(), "Synchronous error when connecting to the |Echo| protocol");

  fidl::SyncClient client{std::move(*client_end)};
  fidl::Result result = client->EchoString({"hello"});
  EXPECT_TRUE(result.is_ok(), "EchoString failed: ");

  const std::string& reply_string = result->response();
  FX_LOGS(INFO) << "Got response: " << reply_string;
}

Rust

#[cfg(test)]
mod tests {
    use anyhow::{Context as _, Error};
    use fidl_fuchsia_examples::{EchoMarker, EchoSynchronousProxy};
    use fuchsia_component::client::connect_channel_to_protocol;
    use fuchsia_zircon as zx;

    #[fuchsia::test]
    fn test() -> Result<(), Error> {
        let (server_end, client_end) = zx::Channel::create();
        connect_channel_to_protocol::<EchoMarker>(server_end)
            .context("Failed to connect to echo service")?;
        let echo = EchoSynchronousProxy::new(client_end);

        // Make an EchoString request, with no timeout for receiving the response.
        let res = echo.echo_string("hello", zx::Time::INFINITE)?;
        println!("response: {:?}", res);
        Ok(())
    }
}

5. Running the test

These instructions require you to open several terminal tabs. Follow the insructions for each tab, from left to right:

Build Fuchsia

fx set core.x64 --with //sdk/ctf/tests/fidl/fuchsia.examples:tests
fx build

Run the emulator

ffx emu start --headless

Serve packages

fx serve

Stream logs

ffx log

Run the tests

fx test -v fuchsia.examples.echo_test
# -v enables verbose output.

If you need additional help debugging at this step, please reach out to fuchsia-ctf-team@google.com.

6. Submit the changes

If the tests pass, send your changes for review. After submission the tests willji automatically be included in the next CTF release when the next milestone branch is cut.

Testing experimental FIDL APIs

You can follow this guide to write a CTF test for a FIDL API that is not in the partner SDK category, but you must not release the test in CTF until the API has been added to the partner category. To prevent the test from being released, set release_in_ctf = false on the test package:

ctf_fuchsia_test_package("experimental-pkg") {
  release_in_ctf = false
  package_name = "experimental-cts-pkg"
  test_components = [ ":experimental" ]
}

See Also

The FAQ sections about retiring tests and disabling tests.