Google celebrates Hispanic Heritage Month. See how.

FIDL case study: key-value store

In this example, we start by creating a simple write-only key-value store, then proceed to augment its functionality with various capabilities, such as reading from the store, iterating over its members, and creating backups.

Creating a write-only key-value store

This baseline case showcases an end-to-end implementation of a write-only key-value store implemented using FIDL. This allows each client to have its own bespoke key-value store on the server, which it can connect to using the Store protocol, and write to using the WriteItem method.

First, we need to define our interface definitions and test harness. The FIDL, CML, and realm interface definitions set up a scaffold that arbitrary implementations can use:

FIDL

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
library examples.keyvaluestore;

/// An item in the store. The key must match the regex `^[A-z][A-z0-9_\.\/]{2,62}[A-z0-9]$`. That
/// is, it must start with a letter, end with a letter or number, contain only letters, numbers,
/// periods, and slashes, and be between 4 and 64 characters long.
type Item = struct {
    key string:128;
    value vector<byte>:64000;
};

/// An enumeration of things that may go wrong when trying to write a value to our store.
type WriteError = flexible enum {
    UNKNOWN = 0;
    INVALID_KEY = 1;
    INVALID_VALUE = 2;
    ALREADY_EXISTS = 3;
};

/// A very basic key-value store - so basic, in fact, that one may only write to it, never read!
@discoverable
protocol Store {
    /// Writes an item to the store.
    WriteItem(struct {
        attempt Item;
    }) -> (struct {}) error WriteError;
};

CML

Client

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    include: [
        // TODO(fxbug.dev/109273): remove this line when the linked bug is resolved.
        "inspect/client.shard.cml",
        "syslog/client.shard.cml",
    ],
    program: {
        runner: "elf",
        binary: "bin/client_bin",
    },
    use: [
        { protocol: "examples.keyvaluestore.Store" },
    ],
    config: {
        write_items: {
            type: "vector",
            max_count: 16,
            element: {
                type: "string",
                max_size: 64,
            },
        },
    },
}

Server

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    include: [ "syslog/client.shard.cml" ],
    program: {
        runner: "elf",
        binary: "bin/server_bin",
    },
    capabilities: [
        { protocol: "examples.keyvaluestore.Store" },
    ],
    expose: [
        {
            protocol: "examples.keyvaluestore.Store",
            from: "self",
        },
    ],
}

Realm

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    children: [
        {
            name: "client",
            url: "#meta/client.cm",
        },
        {
            name: "server",
            url: "#meta/server.cm",
        },
    ],
    offer: [
        // Route the protocol under test from the server to the client.
        {
            protocol: "examples.keyvaluestore.Store",
            from: "#server",
            to: "#client",
        },

        // Route logging support to all children.
        {
            protocol: "fuchsia.logger.LogSink",
            from: "parent",
            to: [
                "#client",
                "#server",
            ],
        },
    ],
}

Client and server implementations can then be written in any supported language:

Rust

Client

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use {
    anyhow::{Context as _, Error},
    config::Config,
    fidl_examples_keyvaluestore::{Item, StoreMarker, StoreSynchronousProxy},
    fuchsia_component::client::connect_channel_to_protocol,
    fuchsia_zircon as zx,
    std::{thread, time},
};

fn main() -> Result<(), Error> {
    println!("Started");

    // Load the structured config values passed to this component at startup.
    let config = Config::take_from_startup_handle();

    // Create a pair of channels endpoints. The `client_end` stays here, while the `server_end` gets
    // sent via the Component Framework runtime to the newly spun up server component. We wrap
    // `client_end` in a proxy object that lets us synchronously send Store requests across the
    // channel.
    let (client_end, server_end) = zx::Channel::create()?;
    let store = StoreSynchronousProxy::new(client_end);
    connect_channel_to_protocol::<StoreMarker>(server_end)
        .context("Failed to connect to Store service")?;
    println!("Outgoing connection enabled");

    // This client's structured config has one parameter, a vector of strings. Each string is the
    // path to a resource file whose filename is a key and whose contents are a value. We iterate
    // over them and try to write each key-value pair to the remote store.
    for key in config.write_items.into_iter() {
        let path = format!("/pkg/data/{}.txt", key);
        let value = std::fs::read_to_string(path.clone())
            .with_context(|| format!("Failed to load {path}"))?;
        let res =
            store.write_item(&mut Item { key: key, value: value.into_bytes() }, zx::Time::INFINITE);
        match res? {
            Ok(_) => println!("WriteItem Success"),
            Err(err) => println!("WriteItem Error: {}", err.into_primitive()),
        }
    }

    // TODO(fxbug.dev/76579): We need to sleep here to make sure all logs get drained. Once the
    // referenced bug has been resolved, we can remove the sleep.
    thread::sleep(time::Duration::from_secs(2));
    Ok(())
}

Server

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use {
    anyhow::{Context as _, Error},
    fidl_examples_keyvaluestore::{Item, StoreRequest, StoreRequestStream, WriteError},
    fuchsia_component::server::ServiceFs,
    futures::prelude::*,
    lazy_static::lazy_static,
    regex::Regex,
    std::cell::RefCell,
    std::collections::hash_map::Entry,
    std::collections::HashMap,
};

lazy_static! {
    static ref KEY_VALIDATION_REGEX: Regex =
        Regex::new(r"^[A-Za-z][A-Za-z0-9_\./]{2,62}[A-Za-z0-9]$")
            .expect("Key validation regex failed to compile");
}

/// Handler for the `WriteItem` method.
fn write_item(store: &mut HashMap<String, Vec<u8>>, attempt: Item) -> Result<(), WriteError> {
    // Validate the key.
    if !KEY_VALIDATION_REGEX.is_match(attempt.key.as_str()) {
        println!("Write error: INVALID_KEY, For key: {}", attempt.key);
        return Err(WriteError::InvalidKey);
    }

    // Validate the value.
    if attempt.value.is_empty() {
        println!("Write error: INVALID_VALUE, For key: {}", attempt.key);
        return Err(WriteError::InvalidValue);
    }

    // Write to the store, validating that the key did not already exist.
    match store.entry(attempt.key) {
        Entry::Occupied(entry) => {
            println!("Write error: ALREADY_EXISTS, For key: {}", entry.key());
            Err(WriteError::AlreadyExists)
        }
        Entry::Vacant(entry) => {
            println!("Wrote value at key: {}", entry.key());
            entry.insert(attempt.value);
            Ok(())
        }
    }
}

/// Creates a new instance of the server. Each server has its own bespoke, per-connection instance
/// of the key-value store.
async fn run_server(stream: StoreRequestStream) -> Result<(), Error> {
    // Create a new in-memory key-value store. The store will live for the lifetime of the
    // connection between the server and this particular client.
    let store = RefCell::new(HashMap::<String, Vec<u8>>::new());

    // Serve all requests on the protocol sequentially - a new request is not handled until its
    // predecessor has been processed.
    stream
        .map(|result| result.context("failed request"))
        .try_for_each(|request| async {
            // Match based on the method being invoked.
            match request {
                StoreRequest::WriteItem { attempt, responder } => {
                    println!("WriteItem request received");

                    // The `responder` parameter is a special struct that manages the outgoing reply
                    // to this method call. Calling `send` on the responder exactly once will send
                    // the reply.
                    responder
                        .send(&mut write_item(&mut store.borrow_mut(), attempt))
                        .context("error sending reply")?;
                    println!("WriteItem response sent");
                }
            }
            Ok(())
        })
        .await
}

// A helper enum that allows us to treat a `Store` service instance as a value.
enum IncomingService {
    Store(StoreRequestStream),
}

#[fuchsia::main]
async fn main() -> Result<(), Error> {
    println!("Started");

    // Add a discoverable instance of our `Store` protocol - this will allow the client to see the
    // server and connect to it.
    let mut fs = ServiceFs::new_local();
    fs.dir("svc").add_fidl_service(IncomingService::Store);
    fs.take_and_serve_directory_handle()?;
    println!("Listening for incoming connections");

    // The maximum number of concurrent clients that may be served by this process.
    const MAX_CONCURRENT: usize = 10;

    // Serve each connection simultaneously, up to the `MAX_CONCURRENT` limit.
    fs.for_each_concurrent(MAX_CONCURRENT, |IncomingService::Store(stream)| {
        run_server(stream).unwrap_or_else(|e| println!("{:?}", e))
    })
    .await;

    Ok(())
}

C++ (Natural)

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Natural) implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Natural) implementation.

C++ (Wire)

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Wire) implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Wire) implementation.

Dart

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): Dart implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): Dart implementation.

HLCPP

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): HLCPP implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): HLCPP implementation.

Adding support for reading from the store

The originally write-only key-value store is now extended with the ability to read items back out of the store.

The changes applied to the FIDL and CML definitions are as follows:

FIDL

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
library examples.keyvaluestore.addreaditem;

// Aliases for the key and value. Using aliases helps increase the readability of FIDL files and
// reduces likelihood of errors due to differing constraints.
alias Key = string:128;
alias Value = vector<byte>:64000;

/// An item in the store. The key must match the regex `^[A-z][A-z0-9_\.\/]{2,62}[A-z0-9]$`. That
/// is, it must start with a letter, end with a letter or number, contain only letters, numbers,
/// periods, and slashes, and be between 4 and 64 characters long.
type Item = struct {
    key Key;
    value Value;
};

/// An enumeration of things that may go wrong when trying to write a value to our store.
type WriteError = flexible enum {
    UNKNOWN = 0;
    INVALID_KEY = 1;
    INVALID_VALUE = 2;
    ALREADY_EXISTS = 3;
};

/// An enumeration of things that may go wrong when trying to read a value out of our store.
type ReadError = flexible enum {
    UNKNOWN = 0;
    NOT_FOUND = 1;
};

/// A very basic key-value store - so basic, in fact, that one may only write to it, never read!
@discoverable
protocol Store {
    /// Writes an item to the store.
    WriteItem(struct {
        attempt Item;
    }) -> (struct {}) error WriteError;

    /// Reads an item from the store.
    ReadItem(struct {
        key Key;
    }) -> (Item) error ReadError;
};

CML

Client

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    include: [
        // TODO(fxbug.dev/109273): remove this line when the linked bug is resolved.
        "inspect/client.shard.cml",
        "syslog/client.shard.cml",
    ],
    program: {
        runner: "elf",
        binary: "bin/client_bin",
    },
    use: [
        { protocol: "examples.keyvaluestore.addreaditem.Store" },
    ],
    config: {
        write_items: {
            type: "vector",
            max_count: 16,
            element: {
                type: "string",
                max_size: 64,
            },
        },

        read_items: {
            type: "vector",
            max_count: 16,
            element: {
                type: "string",
                max_size: 64,
            },
        },

    },
}

Server

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    include: [ "syslog/client.shard.cml" ],
    program: {
        runner: "elf",
        binary: "bin/server_bin",
    },
    capabilities: [
        { protocol: "examples.keyvaluestore.addreaditem.Store" },
    ],
    expose: [
        {
            protocol: "examples.keyvaluestore.addreaditem.Store",
            from: "self",
        },
    ],
}

Realm

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
{
    children: [
        {
            name: "client",
            url: "#meta/client.cm",
        },
        {
            name: "server",
            url: "#meta/server.cm",
        },
    ],
    offer: [
        // Route the protocol under test from the server to the client.
        {
            protocol: "examples.keyvaluestore.addreaditem.Store",
            from: "#server",
            to: "#client",
        },

        // Route logging support to all children.
        {
            protocol: "fuchsia.logger.LogSink",
            from: "parent",
            to: [
                "#client",
                "#server",
            ],
        },
    ],
}

Client and server implementations for all languages change as well:

Rust

Client

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use {
    anyhow::{Context as _, Error},
    config::Config,
    fidl_examples_keyvaluestore_addreaditem::{Item, StoreMarker, StoreSynchronousProxy},
    fuchsia_component::client::connect_channel_to_protocol,
    fuchsia_zircon as zx,
    std::{str, thread, time},
};

fn main() -> Result<(), Error> {
    println!("Started");

    // Load the structured config values passed to this component at startup.
    let config = Config::take_from_startup_handle();

    // Create a pair of channels endpoints. The `client_end` stays here, while the `server_end` gets
    // sent via the Component Framework runtime to the newly spun up server component. We wrap
    // `client_end` in a proxy object that lets us synchronously send Store requests across the
    // channel.
    let (client_end, server_end) = zx::Channel::create()?;
    let store = StoreSynchronousProxy::new(client_end);
    connect_channel_to_protocol::<StoreMarker>(server_end)
        .context("Failed to connect to Store service")?;
    println!("Outgoing connection enabled");

    // This client's structured config has one parameter, a vector of strings. Each string is the
    // path to a resource file whose filename is a key and whose contents are a value. We iterate
    // over them and try to write each key-value pair to the remote store.
    for key in config.write_items.into_iter() {
        let path = format!("/pkg/data/{}.txt", key);
        let value = std::fs::read_to_string(path.clone())
            .with_context(|| format!("Failed to load {path}"))?;
        let res =
            store.write_item(&mut Item { key: key, value: value.into_bytes() }, zx::Time::INFINITE);
        match res? {
            Ok(_) => println!("WriteItem Success"),
            Err(err) => println!("WriteItem Error: {}", err.into_primitive()),
        }
    }

    // The structured config for this client contains `read_items`, a vector of strings, each of
    // which is meant to be read from the key-value store. We iterate over these keys, attempting to
    // read them in turn.
    for key in config.read_items.into_iter() {
        let res = store.read_item(key.as_str(), zx::Time::INFINITE);
        match res.unwrap() {
            Ok(val) => {
                println!("ReadItem Success: key: {}, value: {}", key, str::from_utf8(&val.1)?)
            }
            Err(err) => println!("ReadItem Error: {}", err.into_primitive()),
        }
    }

    // TODO(fxbug.dev/76579): We need to sleep here to make sure all logs get drained. Once the
    // referenced bug has been resolved, we can remove the sleep.
    thread::sleep(time::Duration::from_secs(2));
    Ok(())
}

Server

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use {
    anyhow::{Context as _, Error},
    fidl_examples_keyvaluestore_addreaditem::{
        Item, ReadError, StoreRequest, StoreRequestStream, WriteError,
    },
    fuchsia_component::server::ServiceFs,
    futures::prelude::*,
    lazy_static::lazy_static,
    regex::Regex,
    std::cell::RefCell,
    std::collections::hash_map::Entry,
    std::collections::HashMap,
};

lazy_static! {
    static ref KEY_VALIDATION_REGEX: Regex =
        Regex::new(r"^[A-Za-z][A-Za-z0-9_\./]{2,62}[A-Za-z0-9]$")
            .expect("Key validation regex failed to compile");
}

/// Handler for the `WriteItem` method.
fn write_item(store: &mut HashMap<String, Vec<u8>>, attempt: Item) -> Result<(), WriteError> {
    // Validate the key.
    if !KEY_VALIDATION_REGEX.is_match(attempt.key.as_str()) {
        println!("Write error: INVALID_KEY, For key: {}", attempt.key);
        return Err(WriteError::InvalidKey);
    }

    // Validate the value.
    if attempt.value.is_empty() {
        println!("Write error: INVALID_VALUE, For key: {}", attempt.key);
        return Err(WriteError::InvalidValue);
    }

    // Write to the store, validating that the key did not already exist.
    match store.entry(attempt.key) {
        Entry::Occupied(entry) => {
            println!("Write error: ALREADY_EXISTS, For key: {}", entry.key());
            Err(WriteError::AlreadyExists)
        }
        Entry::Vacant(entry) => {
            println!("Wrote value at key: {}", entry.key());
            entry.insert(attempt.value);
            Ok(())
        }
    }
}

/// Creates a new instance of the server. Each server has its own bespoke, per-connection instance
/// of the key-value store.
async fn run_server(stream: StoreRequestStream) -> Result<(), Error> {
    // Create a new in-memory key-value store. The store will live for the lifetime of the
    // connection between the server and this particular client.
    let store = RefCell::new(HashMap::<String, Vec<u8>>::new());

    // Serve all requests on the protocol sequentially - a new request is not handled until its
    // predecessor has been processed.
    stream
        .map(|result| result.context("failed request"))
        .try_for_each(|request| async {
            // Match based on the method being invoked.
            match request {
                StoreRequest::WriteItem { attempt, responder } => {
                    println!("WriteItem request received");

                    // The `responder` parameter is a special struct that manages the outgoing reply
                    // to this method call. Calling `send` on the responder exactly once will send
                    // the reply.
                    responder
                        .send(&mut write_item(&mut store.borrow_mut(), attempt))
                        .context("error sending reply")?;
                    println!("WriteItem response sent");
                }
                StoreRequest::ReadItem { key, responder } => {
                    println!("ReadItem request received");

                    // Read the item from the store, returning the appropriate error if it could not be found.
                    responder
                        .send(&mut match store.borrow().get(&key) {
                            Some(found) => {
                                println!("Read value at key: {}", key);
                                Ok((key, found.clone()))
                            }
                            None => {
                                println!("Read error: NOT_FOUND, For key: {}", key);
                                Err(ReadError::NotFound)
                            }
                        })
                        .context("error sending reply")?;
                    println!("ReadItem response sent");
            }
            Ok(())
        })
        .await
}

// A helper enum that allows us to treat a `Store` service instance as a value.
enum IncomingService {
    Store(StoreRequestStream),
}

#[fuchsia::main]
async fn main() -> Result<(), Error> {
    println!("Started");

    // Add a discoverable instance of our `Store` protocol - this will allow the client to see the
    // server and connect to it.
    let mut fs = ServiceFs::new_local();
    fs.dir("svc").add_fidl_service(IncomingService::Store);
    fs.take_and_serve_directory_handle()?;
    println!("Listening for incoming connections");

    // The maximum number of concurrent clients that may be served by this process.
    const MAX_CONCURRENT: usize = 10;

    // Serve each connection simultaneously, up to the `MAX_CONCURRENT` limit.
    fs.for_each_concurrent(MAX_CONCURRENT, |IncomingService::Store(stream)| {
        run_server(stream).unwrap_or_else(|e| println!("{:?}", e))
    })
    .await;

    Ok(())
}

C++ (Natural)

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Natural) implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Natural) implementation.

C++ (Wire)

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Wire) implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): C++ (Wire) implementation.

Dart

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): Dart implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): Dart implementation.

HLCPP

Client

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): HLCPP implementation.

Server

<!-- TODO(fxbug.dev/109276): Remove this file once this impl is done. -->

// TODO(fxbug.dev/109276): HLCPP implementation.