A. Examples

A.1. General examples

The following examples show the fundamental concepts for using Model Runtime.

A.1.1. ModelRunner examples (high-level API)

These examples, using the ModelRunner class, load the model from PopEF files and send one inference request to the IPU.

Listing A.1 model_runner_quick_start.cpp
 1int main(int argc, char *argv[]) {
 2  using namespace std::chrono_literals;
 3  static const char *example_desc = "Model runner simple example.";
 4  const boost::program_options::variables_map vm =
 5      examples::parsePopefProgramOptions(example_desc, argc, argv);
 6  const auto popef_paths = vm["popef"].as<std::vector<std::string>>();
 7
 8  model_runtime::ModelRunnerConfig config;
 9  config.device_wait_config =
10      model_runtime::DeviceWaitConfig{600s /*timeout*/, 1s /*sleep_time*/};
11  model_runtime::ModelRunner model_runner(popef_paths, config);
12
13  examples::print("Allocating input tensors");
14
15  const model_runtime::InputMemory input_memory =
16      examples::allocateHostInputData(model_runner.getExecuteInputs());
17
18  examples::printInputMemory(input_memory);
19
20  examples::print("Sending single synchronous request with empty data.");
21
22  const model_runtime::OutputMemory output_memory =
23      model_runner.execute(examples::toInputMemoryView(input_memory));
24
25  examples::print("Received output:");
26
27  using OutputValueType =
28      std::pair<const std::string, model_runtime::TensorMemory>;
29
30  for (const OutputValueType &name_with_memory : output_memory) {
31    auto &&[name, memory] = name_with_memory;
32    examples::print(fmt::format("Output tensor {}, {} bytes", name,
33                                memory.data_size_bytes));
34  }
35
36  examples::print("Success: exiting");
37  return EXIT_SUCCESS;
38}

Download model_runner_quick_start.cpp

A.1.2. Session examples (low-level API)

These examples, using the Session class, load the model from PopEF files and send one inference request to the IPU.

Listing A.3 session_quick_start.cpp
 1int main(int argc, char *argv[]) {
 2
 3  using namespace std::chrono_literals;
 4  static const char *example_desc = "Session quick start example.";
 5  const boost::program_options::variables_map vm =
 6      examples::parsePopefProgramOptions(example_desc, argc, argv);
 7  const auto popef_paths = vm["popef"].as<std::vector<std::string>>();
 8
 9  std::shared_ptr<popef::Model> model = examples::createPopefModel(popef_paths);
10
11  model_runtime::Session session{model};
12  examples::print("Created Session.");
13
14  model_runtime::DeviceManager dm;
15  model_runtime::DeviceWaitConfig wait_config{600s /*timeout*/,
16                                              1s /*sleep_time*/};
17
18  std::shared_ptr<model_runtime::Device> device =
19      dm.getDevice(model, wait_config);
20  examples::print(fmt::format("Device acquired: {}", *device));
21
22  session.bindToDevice(device);
23
24  model_runtime::QueueManager *queue_manager = session.createQueueManager();
25  examples::print("Created QueueManager.");
26
27  using byte_t = unsigned char;
28  using memory_t = std::vector<byte_t>;
29
30  std::vector<memory_t> allocated_memory;
31
32  for (const popef::Anchor *input_anchor : session.getUserInputAnchors()) {
33    const std::string anchor_name = input_anchor->name();
34    const std::size_t size_in_bytes = input_anchor->tensorInfo().sizeInBytes();
35
36    examples::print(
37        fmt::format("Allocating input memory {} bytes for input anchor `{}`",
38                    size_in_bytes, anchor_name));
39    memory_t &memory = allocated_memory.emplace_back(size_in_bytes);
40
41    examples::print(
42        fmt::format("Queue Manager - enqueue input anchor `{}`", anchor_name));
43    queue_manager->inputQueue(anchor_name)
44        .enqueue(memory.data(), size_in_bytes);
45  }
46
47  for (const popef::Anchor *output_anchor : session.getUserOutputAnchors()) {
48    const std::string anchor_name = output_anchor->name();
49    const std::size_t size_in_bytes = output_anchor->tensorInfo().sizeInBytes();
50
51    examples::print(
52        fmt::format("Allocating output memory {} bytes for output anchor `{}`",
53                    size_in_bytes, anchor_name));
54    memory_t &memory = allocated_memory.emplace_back(size_in_bytes);
55    examples::print(fmt::format("{}", memory.size()));
56    examples::print(
57        fmt::format("Queue Manager - enqueue output anchor `{}`", anchor_name));
58    queue_manager->outputQueue(anchor_name)
59        .enqueue(memory.data(), size_in_bytes);
60  }
61
62  examples::print("Session started model execution.");
63  session.runLoadPrograms();
64  session.runMainPrograms();
65  examples::print("Session finished model execution.");
66
67  examples::print("Success: exiting");
68  return EXIT_SUCCESS;
69}
70
71namespace examples {
72
73std::shared_ptr<popef::Model>
74createPopefModel(const std::vector<std::string> &popef_paths) {
75  auto reader = std::make_shared<popef::Reader>();
76  for (const auto &path : popef_paths)
77    reader->parseFile(path);
78
79  return popef::ModelBuilder(reader).createModel();
80}
81
82} // namespace examples

Download session_quick_start.cpp

A.2. Helper functions used by the examples

A file with functions used by the examples.

Download utils.hpp

A.3. Generating an example PopEF file

This program generates a PopEF file used by the examples.

Download generate_simple_popef.cpp