gem5
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
dist_iface.hh
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015-2016 ARM Limited
3  * All rights reserved
4  *
5  * The license below extends only to copyright in the software and shall
6  * not be construed as granting a license to any other intellectual
7  * property including but not limited to intellectual property relating
8  * to a hardware implementation of the functionality of the software
9  * licensed hereunder. You may use the software subject to the license
10  * terms below provided that you ensure that this notice is replicated
11  * unmodified and in its entirety in all distributions of the software,
12  * modified or unmodified, in source code or in binary form.
13  *
14  * Redistribution and use in source and binary forms, with or without
15  * modification, are permitted provided that the following conditions are
16  * met: redistributions of source code must retain the above copyright
17  * notice, this list of conditions and the following disclaimer;
18  * redistributions in binary form must reproduce the above copyright
19  * notice, this list of conditions and the following disclaimer in the
20  * documentation and/or other materials provided with the distribution;
21  * neither the name of the copyright holders nor the names of its
22  * contributors may be used to endorse or promote products derived from
23  * this software without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
28  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
29  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
30  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
31  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
32  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
33  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
34  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
35  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36  *
37  * Authors: Gabor Dozsa
38  */
39 
40 /* @file
41  * The interface class for dist gem5 simulations.
42  *
43  * dist-gem5 is an extension to gem5 to enable parallel simulation of a
44  * distributed system (e.g. simulation of a pool of machines
45  * connected by Ethernet links). A dist gem5 run consists of seperate gem5
46  * processes running in parallel. Each gem5 process executes
47  * the simulation of a component of the simulated distributed system.
48  * (An example component can be a dist-core board with an Ethernet NIC.)
49  * The DistIface class below provides services to transfer data and
50  * control messages among the gem5 processes. The main such services are
51  * as follows.
52  *
53  * 1. Send a data packet coming from a simulated Ethernet link. The packet
54  * will be transferred to (all) the target(s) gem5 processes. The send
55  * operation is always performed by the simulation thread, i.e. the gem5
56  * thread that is processing the event queue associated with the simulated
57  * Ethernet link.
58  *
59  * 2. Spawn a receiver thread to process messages coming in from the
60  * from other gem5 processes. Each simulated Ethernet link has its own
61  * associated receiver thread. The receiver thread saves the incoming packet
62  * and schedule an appropriate receive event in the event queue.
63  *
64  * 3. Schedule a global barrier event periodically to keep the gem5
65  * processes in sync.
66  * Periodic barrier event to keep peer gem5 processes in sync. The basic idea
67  * is that no gem5 process can go ahead further than the simulated link
68  * transmission delay to ensure that a corresponding receive event can always
69  * be scheduled for any message coming in from a peer gem5 process.
70  *
71  *
72  *
73  * This interface is an abstract class. It can work with various low level
74  * send/receive service implementations (e.g. TCP/IP, MPI,...). A TCP
75  * stream socket version is implemented in src/dev/net/tcp_iface.[hh,cc].
76  */
77 #ifndef __DEV_DIST_IFACE_HH__
78 #define __DEV_DIST_IFACE_HH__
79 
80 #include <array>
81 #include <mutex>
82 #include <queue>
83 #include <thread>
84 #include <utility>
85 
86 #include "base/misc.hh"
87 #include "dev/net/dist_packet.hh"
88 #include "dev/net/etherpkt.hh"
89 #include "sim/core.hh"
90 #include "sim/drain.hh"
91 #include "sim/global_event.hh"
92 #include "sim/serialize.hh"
93 
94 class EventManager;
95 class System;
96 class ThreadContext;
97 
101 class DistIface : public Drainable, public Serializable
102 {
103  public:
105 
106  protected:
109 
110  private:
111  class SyncEvent;
118  class Sync : public Serializable
119  {
120  protected:
124  std::mutex lock;
130  std::condition_variable cv;
135  unsigned waitNum;
139  bool doExit;
143  bool doCkpt;
159  bool isAbort;
160 
161  friend class SyncEvent;
162 
163  public:
171  void init(Tick start, Tick repeat);
177  virtual bool run(bool same_tick) = 0;
184  virtual bool progress(Tick send_tick,
185  Tick next_repeat,
186  ReqType do_ckpt,
187  ReqType do_exit,
188  ReqType do_stop_sync) = 0;
193  void abort();
194 
195  virtual void requestCkpt(ReqType req) = 0;
196  virtual void requestExit(ReqType req) = 0;
197  virtual void requestStopSync(ReqType req) = 0;
198 
199  void drainComplete();
200 
201  virtual void serialize(CheckpointOut &cp) const override = 0;
202  virtual void unserialize(CheckpointIn &cp) override = 0;
203  };
204 
205  class SyncNode: public Sync
206  {
207  private:
220 
221  public:
222 
223  SyncNode();
225  bool run(bool same_tick) override;
226  bool progress(Tick max_req_tick,
227  Tick next_repeat,
228  ReqType do_ckpt,
229  ReqType do_exit,
230  ReqType do_stop_sync) override;
231 
232  void requestCkpt(ReqType req) override;
233  void requestExit(ReqType req) override;
234  void requestStopSync(ReqType req) override;
235 
236  void serialize(CheckpointOut &cp) const override;
237  void unserialize(CheckpointIn &cp) override;
238  };
239 
240  class SyncSwitch: public Sync
241  {
242  private:
246  unsigned numExitReq;
250  unsigned numCkptReq;
254  unsigned numStopSyncReq;
258  unsigned numNodes;
259 
260  public:
261  SyncSwitch(int num_nodes);
263 
264  bool run(bool same_tick) override;
265  bool progress(Tick max_req_tick,
266  Tick next_repeat,
267  ReqType do_ckpt,
268  ReqType do_exit,
269  ReqType do_stop_sync) override;
270 
271  void requestCkpt(ReqType) override {
272  panic("Switch requested checkpoint");
273  }
274  void requestExit(ReqType) override {
275  panic("Switch requested exit");
276  }
277  void requestStopSync(ReqType) override {
278  panic("Switch requested stop sync");
279  }
280 
281  void serialize(CheckpointOut &cp) const override;
282  void unserialize(CheckpointIn &cp) override;
283  };
284 
297  class SyncEvent : public GlobalSyncEvent
298  {
299  private:
303  bool _draining;
304  public:
310 
315  void start();
321  void process() override;
322 
323  bool draining() const { return _draining; }
324  void draining(bool fl) { _draining = fl; }
325  };
334  {
335  private:
341  struct Desc : public Serializable
342  {
346 
347  Desc() : sendTick(0), sendDelay(0) {}
349  packet(p), sendTick(s), sendDelay(d) {}
350  Desc(const Desc &d) :
352 
353  void serialize(CheckpointOut &cp) const override;
354  void unserialize(CheckpointIn &cp) override;
355  };
359  std::queue<Desc> descQueue;
401  Tick calcReceiveTick(Tick send_tick,
402  Tick send_delay,
403  Tick prev_recv_tick);
404 
410 
411  public:
419  prevRecvTick(0), recvDone(nullptr), linkDelay(0),
420  eventManager(em), ckptRestore(false) {}
421 
428  void init(Event *recv_done, Tick link_delay);
440  void pushPacket(EthPacketPtr new_packet,
441  Tick send_tick,
442  Tick send_delay);
443 
444  void serialize(CheckpointOut &cp) const override;
445  void unserialize(CheckpointIn &cp) override;
452  void resumeRecvTicks();
453  };
468  std::thread *recvThread;
477 
478  protected:
482  unsigned rank;
486  unsigned size;
490  static unsigned distIfaceNum;
494  unsigned distIfaceId;
495 
496  bool isMaster;
497 
498  private:
502  static unsigned recvThreadsNum;
506  static Sync *sync;
515  static DistIface *master;
519  static System *sys;
523  static bool isSwitch;
524 
525  private:
532  virtual void sendPacket(const Header &header, const EthPacketPtr &packet) = 0;
537  virtual void sendCmd(const Header &header) = 0;
543  virtual bool recvHeader(Header &header) = 0;
550  virtual void recvPacket(const Header &header, EthPacketPtr &packet) = 0;
554  virtual void initTransport() = 0;
561  void spawnRecvThread(const Event *recv_done, Tick link_delay);
565  void recvThreadFunc(Event *recv_done, Tick link_delay);
566 
567  public:
568 
576  DistIface(unsigned dist_rank,
577  unsigned dist_size,
578  Tick sync_start,
579  Tick sync_repeat,
580  EventManager *em,
581  bool use_pseudo_op,
582  bool is_switch,
583  int num_nodes);
584 
585  virtual ~DistIface();
591  void packetOut(EthPacketPtr pkt, Tick send_delay);
601 
602  DrainState drain() override;
603  void drainResume() override;
604  void init(const Event *e, Tick link_delay);
605  void startup();
606 
607  void serialize(CheckpointOut &cp) const override;
608  void unserialize(CheckpointIn &cp) override;
619  static bool readyToExit(Tick delay);
630  static bool readyToCkpt(Tick delay, Tick period);
634  static uint64_t rankParam();
638  static uint64_t sizeParam();
642  static void toggleSync(ThreadContext *tc);
643  };
644 
645 #endif
bool run(bool same_tick) override
Core method to perform a full dist sync.
Definition: dist_iface.cc:126
void pushPacket(EthPacketPtr new_packet, Tick send_tick, Tick send_delay)
Push a newly arrived packet into the desc queue.
Definition: dist_iface.cc:498
Global events and related declarations.
virtual void sendCmd(const Header &header)=0
Send out a control command to the remote end.
void requestCkpt(ReqType) override
Definition: dist_iface.hh:271
void start()
Schedule the first periodic sync event.
Definition: dist_iface.cc:333
void drainComplete()
Definition: dist_iface.cc:292
void spawnRecvThread(const Event *recv_done, Tick link_delay)
spawn the receiver thread.
Definition: dist_iface.cc:719
virtual void requestExit(ReqType req)=0
void startup()
Definition: dist_iface.cc:814
unsigned numNodes
Number of connected simulated nodes.
Definition: dist_iface.hh:258
DrainState
Object drain/handover states.
Definition: drain.hh:71
virtual void sendPacket(const Header &header, const EthPacketPtr &packet)=0
Send out a data packet to the remote end.
#define panic(...)
Definition: misc.hh:153
DrainState drain() override
Notify an object that it needs to drain its state.
Definition: dist_iface.cc:731
static unsigned recvThreadsNum
Number of receiver threads (in this gem5 process)
Definition: dist_iface.hh:502
void draining(bool fl)
Definition: dist_iface.hh:324
bool doCkpt
Flag is set if taking a ckpt is permitted upon sync completion.
Definition: dist_iface.hh:143
Desc(EthPacketPtr p, Tick s, Tick d)
Definition: dist_iface.hh:348
virtual bool recvHeader(Header &header)=0
Receive a header (i.e.
void abort()
Abort processing an on-going sync event (in case of an error, e.g.
Definition: dist_iface.cc:87
bool progress(Tick max_req_tick, Tick next_repeat, ReqType do_ckpt, ReqType do_exit, ReqType do_stop_sync) override
Callback when the receiver thread gets a sync ack message.
Definition: dist_iface.cc:201
bool run(bool same_tick) override
Core method to perform a full dist sync.
Definition: dist_iface.cc:158
void serialize(CheckpointOut &cp) const override
Serialize an object.
Definition: dist_iface.cc:555
A special global event that synchronizes all threads and forces them to process asynchronously enqueu...
bool syncStartOnPseudoOp
Use pseudoOp to start synchronization.
Definition: dist_iface.hh:476
Tick prevRecvTick
The tick when the most recent receive event was processed.
Definition: dist_iface.hh:367
Bitfield< 2 > em
Definition: misc.hh:603
virtual void serialize(CheckpointOut &cp) const override=0
Serialize an object.
EthPacketPtr popPacket()
Fetch the next packet that is to be received by the simulated network link.
Definition: dist_iface.cc:536
static bool isSwitch
Is this node a switch?
Definition: dist_iface.hh:523
Definition: system.hh:83
void resumeRecvTicks()
Adjust receive ticks for pending packets when restoring from a checkpoint.
Definition: dist_iface.cc:465
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Definition: dist_iface.cc:563
void drainResume() override
Resume execution after a successful drain.
Definition: dist_iface.cc:741
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Definition: dist_iface.cc:768
void serialize(CheckpointOut &cp) const override
Serialize an object.
Definition: dist_iface.cc:306
bool _draining
Flag to set when the system is draining.
Definition: dist_iface.hh:303
bool progress(Tick max_req_tick, Tick next_repeat, ReqType do_ckpt, ReqType do_exit, ReqType do_stop_sync) override
Callback when the receiver thread gets a sync ack message.
Definition: dist_iface.cc:242
std::thread * recvThread
Receiver thread pointer.
Definition: dist_iface.hh:468
static void toggleSync(ThreadContext *tc)
Trigger the master to start/stop synchronization.
Definition: dist_iface.cc:855
ThreadContext is the external interface to all thread state for anything outside of the CPU...
void serialize(CheckpointOut &cp) const override
Serialize an object.
Definition: dist_iface.cc:321
virtual bool progress(Tick send_tick, Tick next_repeat, ReqType do_ckpt, ReqType do_exit, ReqType do_stop_sync)=0
Callback when the receiver thread gets a sync ack message.
virtual void initTransport()=0
Init hook for the underlaying transport.
virtual bool run(bool same_tick)=0
Core method to perform a full dist sync.
Received packet descriptor.
Definition: dist_iface.hh:341
unsigned numExitReq
Counter for recording exit requests.
Definition: dist_iface.hh:246
void requestExit(ReqType) override
Definition: dist_iface.hh:274
static DistIface * master
The very first DistIface object created becomes the master.
Definition: dist_iface.hh:515
Interface for objects that might require draining before checkpointing.
Definition: drain.hh:223
bool ckptRestore
Flag to set if receive ticks for pending packets need to be recalculated due to changed link latencie...
Definition: dist_iface.hh:409
Tick linkDelay
The link delay in ticks for the simulated Ethernet link.
Definition: dist_iface.hh:381
void packetOut(EthPacketPtr pkt, Tick send_delay)
Send out an Ethernet packet.
Definition: dist_iface.cc:651
This class implements global sync operations among gem5 peer processes.
Definition: dist_iface.hh:118
static uint64_t sizeParam()
Getter for the dist size param.
Definition: dist_iface.cc:932
Bitfield< 4 > s
Definition: miscregs.hh:1738
void requestStopSync(ReqType) override
Definition: dist_iface.hh:277
void init(const Event *e, Tick link_delay)
Definition: dist_iface.cc:787
uint64_t Tick
Tick count type.
Definition: types.hh:63
Tick calcReceiveTick(Tick send_tick, Tick send_delay, Tick prev_recv_tick)
Calculate the tick to schedule the next receive done event.
Definition: dist_iface.cc:445
unsigned waitNum
Number of receiver threads that not yet completed the current global synchronisation.
Definition: dist_iface.hh:135
unsigned numCkptReq
Counter for recording ckpt requests.
Definition: dist_iface.hh:250
Tick nextRepeat
The repeat value for the next periodic sync.
Definition: dist_iface.hh:151
Tick syncStart
Tick to schedule the first dist sync event.
Definition: dist_iface.hh:459
void init(Event *recv_done, Tick link_delay)
Initialize network link parameters.
Definition: dist_iface.cc:433
static const Priority Sim_Exit_Pri
If we want to exit on this cycle, it's the very last thing we do.
Definition: eventq.hh:172
Bitfield< 9 > d
Definition: miscregs.hh:1375
static bool readyToCkpt(Tick delay, Tick period)
Initiate taking a checkpoint.
Definition: dist_iface.cc:824
static SyncEvent * syncEvent
The singleton SyncEvent object to schedule periodic dist sync.
Definition: dist_iface.hh:510
std::shared_ptr< EthPacketData > EthPacketPtr
Definition: etherpkt.hh:90
DistIface(unsigned dist_rank, unsigned dist_size, Tick sync_start, Tick sync_repeat, EventManager *em, bool use_pseudo_op, bool is_switch, int num_nodes)
ctor
Definition: dist_iface.cc:606
bool doExit
Flag is set if exit is permitted upon sync completion.
Definition: dist_iface.hh:139
static bool readyToExit(Tick delay)
Initiate the exit from the simulation.
Definition: dist_iface.cc:894
SyncEvent()
Only the firstly instantiated DistIface object will call this constructor.
Definition: dist_iface.hh:309
RecvScheduler(EventManager *em)
Scheduler for the incoming data packets.
Definition: dist_iface.hh:418
bool doStopSync
Flag is set if sync is to stop upon sync completion.
Definition: dist_iface.hh:147
void requestExit(ReqType req) override
Definition: dist_iface.cc:281
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Definition: dist_iface.cc:327
The global event to schedule periodic dist sync.
Definition: dist_iface.hh:297
virtual void unserialize(CheckpointIn &cp) override=0
Unserialize an object.
Tick nextAt
Tick for the next periodic sync (if the event is not scheduled yet)
Definition: dist_iface.hh:155
unsigned numStopSyncReq
Counter for recording stop sync requests.
Definition: dist_iface.hh:254
Basic support for object serialization.
Definition: serialize.hh:220
unsigned size
The number of gem5 processes comprising this dist simulation.
Definition: dist_iface.hh:486
void recvThreadFunc(Event *recv_done, Tick link_delay)
The function executed by a receiver thread.
Definition: dist_iface.cc:673
bool isMaster
Definition: dist_iface.hh:496
std::queue< Desc > descQueue
The queue to store the receive descriptors.
Definition: dist_iface.hh:359
RecvScheduler recvScheduler
Meta information about data packets received.
Definition: dist_iface.hh:472
Bitfield< 9 > e
Definition: miscregs.hh:1376
void process() override
This is a global event so process() will only be called by exactly one simulation thread...
Definition: dist_iface.cc:367
DistHeaderPkt::ReqType ReqType
Definition: dist_iface.hh:108
MsgType
The msg type defines what information a dist header packet carries.
Definition: dist_packet.hh:71
ReqType needStopSync
Sync stop requested.
Definition: dist_iface.hh:219
ReqType needExit
Exit requested.
Definition: dist_iface.hh:211
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Definition: dist_iface.cc:588
void requestStopSync(ReqType req) override
Definition: dist_iface.cc:848
Tick syncRepeat
Frequency of dist sync events in ticks.
Definition: dist_iface.hh:463
static System * sys
System pointer used to wakeup sleeping threads when stopping sync.
Definition: dist_iface.hh:519
DistHeaderPkt::Header Header
Definition: dist_iface.hh:104
std::ostream CheckpointOut
Definition: serialize.hh:67
static Sync * sync
The singleton Sync object to perform dist synchronisation.
Definition: dist_iface.hh:506
void init(Tick start, Tick repeat)
Initialize periodic sync params.
Definition: dist_iface.cc:69
Definition: eventq.hh:185
unsigned distIfaceId
Unique id for the dist link.
Definition: dist_iface.hh:494
static unsigned distIfaceNum
Number of DistIface objects (i.e.
Definition: dist_iface.hh:490
virtual void recvPacket(const Header &header, EthPacketPtr &packet)=0
Receive a packet from the remote end.
void unserialize(CheckpointIn &cp) override
Unserialize an object.
Definition: dist_iface.cc:313
virtual void requestStopSync(ReqType req)=0
void requestCkpt(ReqType req) override
Definition: dist_iface.cc:270
void serialize(CheckpointOut &cp) const override
Serialize an object.
Definition: dist_iface.cc:572
void serialize(CheckpointOut &cp) const override
Serialize an object.
Definition: dist_iface.cc:749
std::condition_variable cv
Condition variable for the simulation thread to wait on until all receiver threads completes the curr...
Definition: dist_iface.hh:130
virtual ~DistIface()
Definition: dist_iface.cc:635
std::mutex lock
The lock to protect access to the Sync object.
Definition: dist_iface.hh:124
EthPacketPtr packetIn()
Fetch the packet scheduled to be received next by the simulated network link.
Definition: dist_iface.hh:600
ReqType needCkpt
Ckpt requested.
Definition: dist_iface.hh:215
bool draining() const
Definition: dist_iface.hh:323
DistHeaderPkt::MsgType MsgType
Definition: dist_iface.hh:107
Event * recvDone
The receive done event for the simulated Ethernet link.
Definition: dist_iface.hh:374
virtual void requestCkpt(ReqType req)=0
The interface class to talk to peer gem5 processes.
Definition: dist_iface.hh:101
Class to encapsulate information about data packets received.
Definition: dist_iface.hh:333
static uint64_t rankParam()
Getter for the dist rank param.
Definition: dist_iface.cc:919
Bitfield< 0 > p
EventManager * eventManager
The event manager associated with the simulated Ethernet link.
Definition: dist_iface.hh:388
bool isAbort
Flag is set if the sync is aborted (e.g.
Definition: dist_iface.hh:159
SyncSwitch(int num_nodes)
Definition: dist_iface.cc:96
unsigned rank
The rank of this process among the gem5 peers.
Definition: dist_iface.hh:482

Generated on Fri Jun 9 2017 13:03:46 for gem5 by doxygen 1.8.6