A. Examples
A.1. General examples
The following examples show the fundamental concepts for using Model Runtime.
A.1.1. ModelRunner examples (high-level API)
These examples, using the ModelRunner class, load the model from PopEF files and send one inference request to the IPU.
1int main(int argc, char *argv[]) {
2 using namespace std::chrono_literals;
3 static const char *example_desc = "Model runner simple example.";
4 const boost::program_options::variables_map vm =
5 examples::parsePopefProgramOptions(example_desc, argc, argv);
6 const auto popef_paths = vm["popef"].as<std::vector<std::string>>();
7
8 model_runtime::ModelRunnerConfig config;
9 config.device_wait_config =
10 model_runtime::DeviceWaitConfig{600s /*timeout*/, 1s /*sleep_time*/};
11 model_runtime::ModelRunner model_runner(popef_paths, config);
12
13 examples::print("Allocating input tensors");
14
15 const model_runtime::InputMemory input_memory =
16 examples::allocateHostInputData(model_runner.getExecuteInputs());
17
18 examples::printInputMemory(input_memory);
19
20 examples::print("Sending single synchronous request with empty data.");
21
22 const model_runtime::OutputMemory output_memory =
23 model_runner.execute(examples::toInputMemoryView(input_memory));
24
25 examples::print("Received output:");
26
27 using OutputValueType =
28 std::pair<const std::string, model_runtime::TensorMemory>;
29
30 for (const OutputValueType &name_with_memory : output_memory) {
31 auto &&[name, memory] = name_with_memory;
32 examples::print(fmt::format("Output tensor {}, {} bytes", name,
33 memory.data_size_bytes));
34 }
35
36 examples::print("Success: exiting");
37 return EXIT_SUCCESS;
38}
1def main():
2 parser = argparse.ArgumentParser("Model runner simple example.")
3 parser.add_argument(
4 "-p",
5 "--popef",
6 type=str,
7 metavar='popef_file_path',
8 help="A collection of PopEF files containing the model.",
9 nargs='+',
10 required=True)
11 args = parser.parse_args()
12
13 # Create model runner
14 config = model_runtime.ModelRunnerConfig()
15 config.device_wait_config = model_runtime.DeviceWaitConfig(
16 model_runtime.DeviceWaitStrategy.WAIT_WITH_TIMEOUT,
17 timeout=timedelta(seconds=600),
18 sleepTime=timedelta(seconds=1))
19
20 print("Creating ModelRunner with", config)
21 runner = model_runtime.ModelRunner(model_runtime.PopefPaths(args.popef),
22 config=config)
23
24 print("Preparing input tensors:")
25 input_descriptions = runner.getExecuteInputs()
26 input_tensors = [
27 np.random.randn(*input_desc.shape).astype(input_desc.numpy_data_type())
28 for input_desc in input_descriptions
29 ]
30 input_view = model_runtime.InputMemoryView()
31
32 for input_desc, input_tensor in zip(input_descriptions, input_tensors):
33 print("\tname:", input_desc.name, "shape:", input_tensor.shape,
34 "dtype:", input_tensor.dtype)
35 input_view[input_desc.name] = input_tensor
36
37 print("Sending single synchronous request with empty data.")
38 result = runner.execute(input_view)
39 output_descriptions = runner.getExecuteOutputs()
40
41 print("Processing output tensors:")
42 for output_desc in output_descriptions:
43 output_tensor = np.frombuffer(
44 result[output_desc.name],
45 dtype=output_desc.numpy_data_type()).reshape(output_desc.shape)
46 print("\tname:", output_desc.name, "shape:", output_tensor.shape,
47 "dtype:", output_tensor.dtype, "\n", output_tensor)
48
49 print("Success: exiting")
50 return 0
51
52
A.1.2. Session examples (low-level API)
These examples, using the Session class, load the model from PopEF files and send one inference request to the IPU.
1int main(int argc, char *argv[]) {
2
3 using namespace std::chrono_literals;
4 static const char *example_desc = "Session quick start example.";
5 const boost::program_options::variables_map vm =
6 examples::parsePopefProgramOptions(example_desc, argc, argv);
7 const auto popef_paths = vm["popef"].as<std::vector<std::string>>();
8
9 std::shared_ptr<popef::Model> model = examples::createPopefModel(popef_paths);
10
11 model_runtime::Session session{model};
12 examples::print("Created Session.");
13
14 model_runtime::DeviceManager dm;
15 model_runtime::DeviceWaitConfig wait_config{600s /*timeout*/,
16 1s /*sleep_time*/};
17
18 std::shared_ptr<model_runtime::Device> device =
19 dm.getDevice(model, wait_config);
20 examples::print(fmt::format("Device acquired: {}", *device));
21
22 session.bindToDevice(device);
23
24 model_runtime::QueueManager *queue_manager = session.createQueueManager();
25 examples::print("Created QueueManager.");
26
27 using byte_t = unsigned char;
28 using memory_t = std::vector<byte_t>;
29
30 std::vector<memory_t> allocated_memory;
31
32 for (const popef::Anchor *input_anchor : session.getUserInputAnchors()) {
33 const std::string anchor_name = input_anchor->name();
34 const std::size_t size_in_bytes = input_anchor->tensorInfo().sizeInBytes();
35
36 examples::print(
37 fmt::format("Allocating input memory {} bytes for input anchor `{}`",
38 size_in_bytes, anchor_name));
39 memory_t &memory = allocated_memory.emplace_back(size_in_bytes);
40
41 examples::print(
42 fmt::format("Queue Manager - enqueue input anchor `{}`", anchor_name));
43 queue_manager->inputQueue(anchor_name)
44 .enqueue(memory.data(), size_in_bytes);
45 }
46
47 for (const popef::Anchor *output_anchor : session.getUserOutputAnchors()) {
48 const std::string anchor_name = output_anchor->name();
49 const std::size_t size_in_bytes = output_anchor->tensorInfo().sizeInBytes();
50
51 examples::print(
52 fmt::format("Allocating output memory {} bytes for output anchor `{}`",
53 size_in_bytes, anchor_name));
54 memory_t &memory = allocated_memory.emplace_back(size_in_bytes);
55 examples::print(fmt::format("{}", memory.size()));
56 examples::print(
57 fmt::format("Queue Manager - enqueue output anchor `{}`", anchor_name));
58 queue_manager->outputQueue(anchor_name)
59 .enqueue(memory.data(), size_in_bytes);
60 }
61
62 examples::print("Session started model execution.");
63 session.runLoadPrograms();
64 session.runMainPrograms();
65 examples::print("Session finished model execution.");
66
67 examples::print("Success: exiting");
68 return EXIT_SUCCESS;
69}
70
71namespace examples {
72
73std::shared_ptr<popef::Model>
74createPopefModel(const std::vector<std::string> &popef_paths) {
75 auto reader = std::make_shared<popef::Reader>();
76 for (const auto &path : popef_paths)
77 reader->parseFile(path);
78
79 return popef::ModelBuilder(reader).createModel();
80}
81
82} // namespace examples
1def main():
2 parser = argparse.ArgumentParser("Model runner simple example.")
3 parser.add_argument(
4 "-p",
5 "--popef",
6 type=str,
7 metavar='popef_file_path',
8 help="A collection of PopEF files containing the model.",
9 nargs='+',
10 required=True)
11 args = parser.parse_args()
12
13 for model_file in args.popef:
14 reader = popef.Reader()
15 reader.parseFile(model_file)
16
17 model = popef.ModelBuilder(reader).createModel()
18
19 dm = model_runtime.DeviceManager()
20 wait_config = model_runtime.DeviceWaitConfig(
21 model_runtime.DeviceWaitStrategy.WAIT_WITH_TIMEOUT,
22 timeout=timedelta(seconds=600),
23 sleepTime=timedelta(seconds=1))
24 device = dm.getDevice(model, wait_config=wait_config)
25 print("Device acquired:", device)
26
27 session = model_runtime.Session(model)
28 print("Created Session.")
29
30 session.bindToDevice(device)
31
32 queue_manager = session.createQueueManager()
33 print("Created QueueManager.")
34
35 user_input_anchors = session.getUserInputAnchors()
36 input_data = prepare_io_memory(user_input_anchors)
37
38 for anchor, tensor in zip(user_input_anchors, input_data):
39 anchor_name = anchor.name()
40 print("Enqueue input anchor ", anchor_name)
41 queue_manager.inputs[anchor_name].enqueue(tensor)
42
43 user_output_anchors = session.getUserOutputAnchors()
44 output_data = prepare_io_memory(user_input_anchors)
45
46 for anchor, tensor in zip(user_output_anchors, output_data):
47 anchor_name = anchor.name()
48 print("Enqueue output anchor ", anchor_name)
49 queue_manager.outputs[anchor_name].enqueue(tensor)
50
51 print("Running load programs")
52 session.runLoadPrograms()
53
54 print("Running main programs")
55 session.runMainPrograms()
56
57 print("Execution finished. Results available.")
58
59 print("Success: exiting")
60 return 0
61
62
63def prepare_io_memory(anchors):
64 data = []
65 for anchor in anchors:
66 info = anchor.tensorInfo()
67 shape = info.shape()
68 data.append(np.random.randn(*shape).astype(info.numpyDType()))
69 return data
70
71
A.2. Helper functions used by the examples
A file with functions used by the examples.
A.3. Generating an example PopEF file
This program generates a PopEF file used by the examples.