Loading [MathJax]/extensions/tex2jax.js
CIRCT 21.0.0git
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
Engines.cpp
Go to the documentation of this file.
1//===- Engines.cpp --------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// DO NOT EDIT!
10// This file is distributed as part of an ESI package. The source for this file
11// should always be modified within CIRCT (lib/dialect/ESI/runtime/cpp/).
12//
13//===----------------------------------------------------------------------===//
14
15#include "esi/Engines.h"
16#include "esi/Accelerator.h"
17
18#include <algorithm>
19#include <cstring>
20
21using namespace esi;
22
23//===----------------------------------------------------------------------===//
24// Unknown Engine
25//===----------------------------------------------------------------------===//
26
27namespace {
28/// Created by default when the DMA engine cannot be resolved. Throws the error
29/// upon trying to connect and creates ports which throw errors on their
30/// connection attempts.
31class UnknownEngine : public Engine {
32public:
33 UnknownEngine(AcceleratorConnection &conn, std::string engineName)
34 : Engine(conn), engineName(engineName) {}
35
36 void connect() override {
37 throw std::runtime_error("Unknown engine '" + engineName + "'");
38 }
39
40 std::unique_ptr<ChannelPort> createPort(AppIDPath idPath,
41 const std::string &channelName,
43 const Type *type) override {
44 if (BundlePort::isWrite(dir))
45 return std::make_unique<UnknownWriteChannelPort>(
46 type,
47 "Unknown engine '" + engineName + "': cannot create write port");
48 else
49 return std::make_unique<UnknownReadChannelPort>(
50 type, "Unknown engine '" + engineName + "': cannot create read port");
51 }
52
53protected:
54 std::string engineName;
55};
56} // namespace
57
58//===----------------------------------------------------------------------===//
59// OneItemBuffersToHost engine
60//
61// Protocol:
62// 1) Host sends address of buffer address via MMIO write.
63// 2) Device writes data on channel with a byte '1' to said buffer address.
64// 3) Host polls the last byte in buffer for '1'.
65// 4) Data is copied out of buffer, last byte is set to '0', goto 1.
66//===----------------------------------------------------------------------===//
67
68namespace {
69class OneItemBuffersToHost;
70class OneItemBuffersToHostReadPort : public ReadChannelPort {
71public:
72 /// Offset into the MMIO space to which the buffer pointer is written.
73 static constexpr size_t BufferPtrOffset = 8;
74
75 OneItemBuffersToHostReadPort(const Type *type, OneItemBuffersToHost *engine,
76 AppIDPath idPath, const std::string &channelName)
77 : ReadChannelPort(type), engine(engine), idPath(idPath),
78 channelName(channelName) {
79 bufferSize = (type->getBitWidth() / 8) + 1;
80 }
81
82 // Write the location of the buffer to the MMIO space.
83 void writeBufferPtr();
84 // Connect allocate a buffer and prime the pump.
85 void connectImpl(std::optional<unsigned>) override;
86 // Check for buffer fill.
87 bool pollImpl() override;
88
89 std::string identifier() const { return idPath.toStr() + "." + channelName; }
90
91protected:
92 // Size of buffer based on type.
93 size_t bufferSize;
94 // Owning engine.
95 OneItemBuffersToHost *engine;
96 // Single buffer.
97 std::unique_ptr<services::HostMem::HostMemRegion> buffer;
98 // Number of times the poll function has been called.
99 uint64_t pollCount = 0;
100
101 // Identifing information.
102 AppIDPath idPath;
103 std::string channelName;
104};
105
106class OneItemBuffersToHost : public Engine {
107 friend class OneItemBuffersToHostReadPort;
108
109public:
110 OneItemBuffersToHost(AcceleratorConnection &conn, AppIDPath idPath,
111 const ServiceImplDetails &details)
112 : Engine(conn), thisPath(idPath) {
113 // Get the MMIO path but don't try to resolve it yet.
114 auto mmioIDIter = details.find("mmio");
115 if (mmioIDIter != details.end())
116 mmioID = std::any_cast<AppID>(mmioIDIter->second);
117 }
118
119 static std::unique_ptr<Engine> create(AcceleratorConnection &conn,
120 AppIDPath idPath,
121 const ServiceImplDetails &details,
122 const HWClientDetails &clients) {
123 return std::make_unique<OneItemBuffersToHost>(conn, idPath, details);
124 }
125
126 // Only throw errors on connect.
127 void connect() override;
128
129 // Create a real read port if requested. Create an error-throwing write port
130 // if requested.
131 std::unique_ptr<ChannelPort> createPort(AppIDPath idPath,
132 const std::string &channelName,
134 const Type *type) override {
135 if (BundlePort::isWrite(dir))
136 return std::make_unique<UnknownWriteChannelPort>(
137 type, idPath.toStr() + "." + channelName +
138 " OneItemBuffersToHost: cannot create write port");
139 return std::make_unique<OneItemBuffersToHostReadPort>(type, this, idPath,
140 channelName);
141 }
142
143protected:
144 AppIDPath thisPath;
145 std::optional<AppID> mmioID;
147 services::HostMem *hostMem;
148};
149} // namespace
150
151void OneItemBuffersToHost::connect() {
152 // This is where we throw errors.
153 if (connected)
154 return;
155 if (!mmioID)
156 throw std::runtime_error("OneItemBuffersToHost: no mmio path specified");
157 hostMem = conn.getService<services::HostMem>();
158 if (!hostMem)
159 throw std::runtime_error("OneItemBuffersToHost: no host memory service");
160 hostMem->start();
161
162 // Resolve the MMIO port.
163 Accelerator &acc = conn.getAccelerator();
164 AppIDPath mmioPath = thisPath;
165 mmioPath.pop_back();
166 mmioPath.push_back(*mmioID);
167 AppIDPath lastPath;
168 BundlePort *port = acc.resolvePort(mmioPath, lastPath);
169 if (port == nullptr)
170 throw std::runtime_error(
171 thisPath.toStr() +
172 " OneItemBuffersToHost: could not find MMIO port at " +
173 mmioPath.toStr());
174 mmio = dynamic_cast<services::MMIO::MMIORegion *>(port);
175 if (!mmio)
176 throw std::runtime_error(
177 thisPath.toStr() +
178 " OneItemBuffersToHost: MMIO port is not an MMIO port");
179
180 // If we have a valid MMIO port, we can connect.
181 connected = true;
182}
183
184void OneItemBuffersToHostReadPort::writeBufferPtr() {
185 uint8_t *bufferData = reinterpret_cast<uint8_t *>(buffer->getPtr());
186 bufferData[bufferSize - 1] = 0;
187 // Write the *device-visible* address of the buffer to the MMIO space.
188 engine->mmio->write(BufferPtrOffset,
189 reinterpret_cast<uint64_t>(buffer->getDevicePtr()));
190}
191
192void OneItemBuffersToHostReadPort::connectImpl(std::optional<unsigned>) {
193 engine->connect();
194 buffer = engine->hostMem->allocate(bufferSize, {});
195 writeBufferPtr();
196}
197
198bool OneItemBuffersToHostReadPort::pollImpl() {
199 // Check to see if the buffer has been filled.
200 uint8_t *bufferData = reinterpret_cast<uint8_t *>(buffer->getPtr());
201 if (bufferData[bufferSize - 1] == 0)
202 return false;
203
204 // If it has, copy the data out. If the consumer (callback) reports that it
205 // has accepted the data, re-use the buffer.
206 MessageData data(bufferData, bufferSize - 1);
207 engine->conn.getLogger().debug(
208 [this, data](std::string &subsystem, std::string &msg,
209 std::unique_ptr<std::map<std::string, std::any>> &details) {
210 subsystem = "OneItemBuffersToHost";
211 msg = identifier() + " got message contents 0x" + data.toHex();
212 });
213 if (callback(std::move(data))) {
214 writeBufferPtr();
215 return true;
216 }
217 return false;
218}
219
220REGISTER_ENGINE("OneItemBuffersToHost", OneItemBuffersToHost);
221
222//===----------------------------------------------------------------------===//
223// OneItemBuffersFromHost engine
224//
225// Protocol:
226// 1) Host allocates two buffers: one for data and one for the completion byte.
227// 2) On write, if the completion byte is '0', the write fails since that
228// implies there's data currently in the buffer which the device has not yet
229// picked up. If the completion byte is '1', continue on.
230// 3) The host writes the data to the buffer and sets the completion byte to
231// '0'.
232// 4) The device reads the data into the data buffer and sets the completion
233// byte to '1'.
234// 5) Host sends address of buffer via MMIO write to register 0x8.
235// 6) Host sends address of completion buffer via MMIO write to register 0x10.
236// 7) Device reads data from said buffer address.
237// 8) Device writes '1' to the first byte of the completion buffer.
238//
239//===----------------------------------------------------------------------===//
240
241namespace {
242
243class OneItemBuffersFromHost;
244class OneItemBuffersFromHostWritePort : public WriteChannelPort {
245public:
246 /// Offset into the MMIO space to which the buffer pointer is written.
247 static constexpr size_t BufferPtrOffset = 8;
248 static constexpr size_t CompletionPtrOffset = 16;
249
250 OneItemBuffersFromHostWritePort(const Type *type,
251 OneItemBuffersFromHost *engine,
252 AppIDPath idPath,
253 const std::string &channelName)
254 : WriteChannelPort(type), engine(engine), idPath(idPath),
255 channelName(channelName) {
256 bufferSize = type->getBitWidth() / 8;
257 }
258
259 // Connect allocate a buffer and prime the pump.
260 void connectImpl(std::optional<unsigned>) override;
261
262 void write(const MessageData &) override;
263 bool tryWrite(const MessageData &data) override;
264
265 std::string identifier() const { return idPath.toStr() + "." + channelName; }
266
267protected:
268 // Size of buffer based on type.
269 size_t bufferSize;
270 // Owning engine.
271 OneItemBuffersFromHost *engine;
272 // Single buffer.
273 std::unique_ptr<services::HostMem::HostMemRegion> data_buffer;
274 std::unique_ptr<services::HostMem::HostMemRegion> completion_buffer;
275
276 // Thread protection.
277 std::mutex bufferMutex;
278
279 // Identifing information.
280 AppIDPath idPath;
281 std::string channelName;
282};
283
284class OneItemBuffersFromHost : public Engine {
285 friend class OneItemBuffersFromHostWritePort;
286
287public:
288 OneItemBuffersFromHost(AcceleratorConnection &conn, AppIDPath idPath,
289 const ServiceImplDetails &details)
290 : Engine(conn), thisPath(idPath) {
291 // Get the MMIO path but don't try to resolve it yet.
292 auto mmioIDIter = details.find("mmio");
293 if (mmioIDIter != details.end())
294 mmioID = std::any_cast<AppID>(mmioIDIter->second);
295 }
296
297 static std::unique_ptr<Engine> create(AcceleratorConnection &conn,
298 AppIDPath idPath,
299 const ServiceImplDetails &details,
300 const HWClientDetails &clients) {
301 return std::make_unique<OneItemBuffersFromHost>(conn, idPath, details);
302 }
303
304 // Only throw errors on connect.
305 void connect() override;
306
307 // Create a real write port if requested. Create an error-throwing read port
308 // if requested.
309 std::unique_ptr<ChannelPort> createPort(AppIDPath idPath,
310 const std::string &channelName,
312 const Type *type) override {
313 if (!BundlePort::isWrite(dir))
314 return std::make_unique<UnknownReadChannelPort>(
315 type, idPath.toStr() + "." + channelName +
316 " OneItemBuffersFromHost: cannot create read port");
317 return std::make_unique<OneItemBuffersFromHostWritePort>(type, this, idPath,
318 channelName);
319 }
320
321protected:
322 AppIDPath thisPath;
323 std::optional<AppID> mmioID;
325 services::HostMem *hostMem;
326};
327
328} // namespace
329
330void OneItemBuffersFromHost::connect() {
331 // This is where we throw errors.
332 if (connected)
333 return;
334 if (!mmioID)
335 throw std::runtime_error("OneItemBuffersFromHost: no mmio path specified");
336 hostMem = conn.getService<services::HostMem>();
337 if (!hostMem)
338 throw std::runtime_error("OneItemBuffersFromHost: no host memory service");
339 hostMem->start();
340
341 // Resolve the MMIO port.
342 Accelerator &acc = conn.getAccelerator();
343 AppIDPath mmioPath = thisPath;
344 mmioPath.pop_back();
345 mmioPath.push_back(*mmioID);
346 AppIDPath lastPath;
347 BundlePort *port = acc.resolvePort(mmioPath, lastPath);
348 if (port == nullptr)
349 throw std::runtime_error(
350 thisPath.toStr() +
351 " OneItemBuffersFromHost: could not find MMIO port at " +
352 mmioPath.toStr());
353 mmio = dynamic_cast<services::MMIO::MMIORegion *>(port);
354 if (!mmio)
355 throw std::runtime_error(
356 thisPath.toStr() +
357 " OneItemBuffersFromHost: MMIO port is not an MMIO port");
358
359 // If we have a valid MMIO port, we can connect.
360 connected = true;
361}
362
363void OneItemBuffersFromHostWritePort::connectImpl(std::optional<unsigned>) {
364 engine->connect();
365 data_buffer =
366 engine->hostMem->allocate(std::max(bufferSize, (size_t)512), {});
367 completion_buffer = engine->hostMem->allocate(512, {});
368 // Set the last byte to '1' to indicate that the buffer is ready to be filled
369 // and sent to the device.
370 *static_cast<uint8_t *>(completion_buffer->getPtr()) = 1;
371}
372
373void OneItemBuffersFromHostWritePort::write(const MessageData &data) {
374 while (!tryWrite(data))
375 // Wait for the device to read the last data we sent.
376 std::this_thread::yield();
377}
378
379bool OneItemBuffersFromHostWritePort::tryWrite(const MessageData &data) {
380 // Check to see if there's an outstanding write.
381 completion_buffer->flush();
382 uint8_t *completion =
383 reinterpret_cast<uint8_t *>(completion_buffer->getPtr());
384 if (*completion == 0)
385 return false;
386
387 // If the buffer is empty, use it.
388 std::lock_guard<std::mutex> lock(bufferMutex);
389 void *bufferData = data_buffer->getPtr();
390 std::memcpy(bufferData, data.getBytes(), data.getSize());
391 data_buffer->flush();
392 // Indicate that the buffer is now in use.
393 *completion = 0;
394 completion_buffer->flush();
395 // Write the *device-visible* address of the data buffer and completion buffer
396 // to the MMIO space.
397 engine->mmio->write(BufferPtrOffset,
398 reinterpret_cast<uint64_t>(data_buffer->getDevicePtr()));
399 engine->mmio->write(
400 CompletionPtrOffset,
401 reinterpret_cast<uint64_t>(completion_buffer->getDevicePtr()));
402 return true;
403}
404
405REGISTER_ENGINE("OneItemBuffersFromHost", OneItemBuffersFromHost);
406
407//===----------------------------------------------------------------------===//
408// Engine / Bundle Engine Map
409//===----------------------------------------------------------------------===//
410
412 const std::string &channelName,
413 BundleType::Direction dir, const Type *type) {
414 auto portIter = ownedPorts.find(std::make_pair(idPath, channelName));
415 if (portIter != ownedPorts.end())
416 return *portIter->second;
417 std::unique_ptr<ChannelPort> port =
418 createPort(idPath, channelName, dir, type);
419 ChannelPort &ret = *port;
420 ownedPorts.emplace(std::make_pair(idPath, channelName), std::move(port));
421 return ret;
422}
423
425 const BundleType *bundleType) const {
426 PortMap ports;
427 for (auto [channelName, dir, type] : bundleType->getChannels()) {
428 auto engineIter = bundleEngineMap.find(channelName);
429 if (engineIter == bundleEngineMap.end())
430 continue;
431
432 ports.emplace(channelName, engineIter->second->requestPort(
433 idPath, channelName, dir, type));
434 }
435 return ports;
436}
437
438void BundleEngineMap::setEngine(const std::string &channelName,
439 Engine *engine) {
440 auto [it, inserted] = bundleEngineMap.try_emplace(channelName, engine);
441 if (!inserted)
442 throw std::runtime_error("Channel already exists in engine map");
443}
444
445//===----------------------------------------------------------------------===//
446// Registry
447//===----------------------------------------------------------------------===//
448
449namespace {
450class EngineRegistry {
451public:
452 static std::map<std::string, registry::internal::EngineCreate> &get() {
453 static EngineRegistry instance;
454 return instance.engineRegistry;
455 }
456
457private:
458 std::map<std::string, registry::internal::EngineCreate> engineRegistry;
459};
460} // namespace
461
462std::unique_ptr<Engine>
464 const std::string &dmaEngineName, AppIDPath idPath,
465 const ServiceImplDetails &details,
466 const HWClientDetails &clients) {
467 auto &reg = EngineRegistry::get();
468 auto it = reg.find(dmaEngineName);
469 if (it == reg.end())
470 return std::make_unique<UnknownEngine>(conn, dmaEngineName);
471 return it->second(conn, idPath, details, clients);
472}
473
474void registry::internal::registerEngine(const std::string &name,
475 EngineCreate create) {
476 auto tried = EngineRegistry::get().try_emplace(name, create);
477 if (!tried.second)
478 throw std::runtime_error("Engine already exists in registry");
479}
#define REGISTER_ENGINE(Name, TEngine)
Definition Engines.h:115
Abstract class representing a connection to an accelerator.
Definition Accelerator.h:79
ServiceClass * getService(AppIDPath id={}, std::string implName={}, ServiceImplDetails details={}, HWClientDetails clients={})
Get a typed reference to a particular service type.
Accelerator & getAccelerator()
Top level accelerator class.
Definition Accelerator.h:60
std::string toStr() const
Definition Manifest.cpp:731
PortMap requestPorts(const AppIDPath &idPath, const BundleType *bundleType) const
Request ports for all the channels in a bundle.
Definition Engines.cpp:424
std::map< std::string, Engine * > bundleEngineMap
Definition Engines.h:86
void setEngine(const std::string &channelName, Engine *engine)
Set a particlar engine for a particular channel.
Definition Engines.cpp:438
Services provide connections to 'bundles' – collections of named, unidirectional communication channe...
Definition Ports.h:226
Bundles represent a collection of channels.
Definition Types.h:44
const ChannelVector & getChannels() const
Definition Types.h:54
Unidirectional channels are the basic communication primitive between the host and accelerator.
Definition Ports.h:36
virtual void connectImpl(std::optional< unsigned > bufferSize)
Called by all connect methods to let backends initiate the underlying connections.
Definition Ports.h:73
virtual bool pollImpl()
Method called by poll() to actually poll the channel if the channel is connected.
Definition Ports.h:69
Engines implement the actual channel communication between the host and the accelerator.
Definition Engines.h:42
virtual void connect()
Start the engine, if applicable.
Definition Engines.h:47
virtual ChannelPort & requestPort(AppIDPath idPath, const std::string &channelName, BundleType::Direction dir, const Type *type)
Get a port for a channel, from the cache if it exists or create it.
Definition Engines.cpp:411
virtual std::unique_ptr< ChannelPort > createPort(AppIDPath idPath, const std::string &channelName, BundleType::Direction dir, const Type *type)=0
Each engine needs to know how to create a ports.
std::map< std::pair< AppIDPath, std::string >, std::unique_ptr< ChannelPort > > ownedPorts
Definition Engines.h:68
BundlePort * resolvePort(const AppIDPath &path, AppIDPath &lastLookup) const
Attempt to resolve a path to a port.
Definition Design.cpp:72
A logical chunk of data representing serialized data.
Definition Common.h:103
A ChannelPort which reads data from the accelerator.
Definition Ports.h:124
Root class of the ESI type system.
Definition Types.h:27
virtual std::ptrdiff_t getBitWidth() const
Definition Types.h:34
A ChannelPort which sends data to the accelerator.
Definition Ports.h:77
A "slice" of some parent MMIO space.
Definition Services.h:159
connect(destination, source)
Definition support.py:39
std::function< std::unique_ptr< Engine >(AcceleratorConnection &conn, AppIDPath idPath, const ServiceImplDetails &details, const HWClientDetails &clients)> EngineCreate
Engines can register themselves for pluggable functionality.
Definition Engines.h:104
void registerEngine(const std::string &name, EngineCreate create)
Definition Engines.cpp:474
std::unique_ptr< Engine > createEngine(AcceleratorConnection &conn, const std::string &dmaEngineName, AppIDPath idPath, const ServiceImplDetails &details, const HWClientDetails &clients)
Create an engine by name.
Definition Engines.cpp:463
Definition esi.py:1
std::map< std::string, std::any > ServiceImplDetails
Definition Common.h:98
std::map< std::string, ChannelPort & > PortMap
Definition Ports.h:29
std::vector< HWClientDetail > HWClientDetails
Definition Common.h:97