Add Virtual SQ (#991)

* mem: add VirtualSQ simulation

set physicalSQ size as 64 and VirtualSQ size as 128.

Change-Id: I40e142d3591f040bc10f8e504562a90455dcef06

* mem: enable phySQFullCheckAtReplay

Change-Id: I0b34a3c8bba373c6b0f94fa203f5942e30b39d78

Author: happy-lx

configs/example/idealkmhv3.py

@@ -94,6 +94,7 @@ def setKmhV3IdealParams(args, system):
         # lsq
         cpu.LQEntries = 120
         cpu.SQEntries = 64
+        cpu.StoreQueueMultiple = 2
         cpu.RARQEntries = 96
         cpu.RAWQEntries = 56
         cpu.LoadCompletionWidth = 8

configs/example/kmhv3.py

@@ -96,6 +96,7 @@ def setKmhV3Params(args, system):
         # lsq
         cpu.LQEntries = 120
         cpu.SQEntries = 64
+        cpu.StoreQueueMultiple = 2
         cpu.RARQEntries = 96
         cpu.RAWQEntries = 56
         cpu.LoadCompletionWidth = 8

src/arch/riscv/insts/mem.cc

@@ -35,6 +35,7 @@
 #include "arch/riscv/insts/static_inst.hh"
 #include "arch/riscv/utility.hh"
 #include "cpu/o3/dyn_inst.hh"
+#include "cpu/o3/inst_queue.hh"
 #include "cpu/o3/lsq_unit.hh"
 #include "cpu/static_inst.hh"
 
@@ -66,6 +67,23 @@ Fault
 StoreData::execute(ExecContext *xc, Trace::InstRecord *) const
 {
     auto inst = dynamic_cast<o3::DynInst *>(xc);
+
+    // STD does not enter the STA store-pipe S0. Consume the physical-SQ
+    // replay marker at the start of each retry before checking the window.
+    if (inst->needPhysicalSQFullReplay()) {
+        inst->clearReplayType();
+        inst->clearReplayFlags();
+        inst->clearNeedReplay();
+    }
+
+    // If no space left in the physical SQ,
+    // then defer the STD operation, will be replayed by IQ.
+    if (!inst->instQueue->storeQueueWriteReady(inst)) {
+        inst->setPhysicalSQFullReplay();
+        inst->instQueue->deferPhysicalSQFullReplay(inst);
+        return NoFault;
+    }
+
     auto data = inst->getRegOperand(inst->staticInst.get(), 0);
 
     if (inst->sqIt->instruction()->getFault() == NoFault) {
@@ -77,6 +95,7 @@ StoreData::execute(ExecContext *xc, Trace::InstRecord *) const
             memcpy(inst->sqIt->instruction()->memData, &data, memsize);
         memcpy(inst->sqIt->data(), &data, memsize);
         inst->sqIt->setStatus(o3::SplitStoreStatus::DataReady);
+        inst->instQueue->recordAddrOrDataReady(inst);
         inst->sqIt->setStatus(o3::SplitStoreStatus::StdPipeFinish);
     }
 

src/cpu/o3/BaseO3CPU.py

@@ -177,7 +177,11 @@ def support_take_over(cls):
             "Time buffer size for forward communication")
 
     LQEntries = Param.Unsigned(72, "Number of load queue entries")
-    SQEntries = Param.Unsigned(56, "Number of store queue entries")
+    SQEntries = Param.Unsigned(56, "Number of physical store queue entries")
+    StoreQueueMultiple = Param.Unsigned(1,
+        "Virtual-to-physical store queue capacity multiplier (power of two)")
+    phySQFullCheckAtReplay = Param.Bool(True,
+        "Wait for physical store queue space before starting a full-SQ replay")
 
     LdPipeStages = Param.Unsigned(4, "Number of load pipeline stages")
     StPipeStages = Param.Unsigned(5, "Number of store pipeline stages")

src/cpu/o3/dyn_inst.cc

@@ -474,6 +474,7 @@ DynInstPtr DynInst::createStoreDataUop()
     stduop->setPredTaken(this->readPredTaken());
     stduop->setTid(this->threadNumber);
     stduop->setThreadState(this->thread);
+    stduop->instQueue = this->instQueue;
 
     // The store-data uop's only source is the original store's data source.
     // For RISC-V stores, src0 is the address base and src1 is the data value.

src/cpu/o3/dyn_inst.hh

@@ -86,6 +86,7 @@ namespace o3
 {
 
 class IssueQue;
+class InstructionQueue;
 
 class DynInst : public ExecContext, public RefCounted
 {
@@ -496,6 +497,7 @@ class DynInst : public ExecContext, public RefCounted
     RequestPtr reqToVerify;
 
     IssueQue* issueQue = nullptr;
+    InstructionQueue* instQueue = nullptr;
     int issueportid = -1;
     int iqtag = -1;
 
@@ -1174,6 +1176,11 @@ class DynInst : public ExecContext, public RefCounted
     void setRescheduleReplay() { setReplay(LdStReplayType::RescheduleReplay); }
     bool needRescheduleReplay() const { return getReplayType() == LdStReplayType::RescheduleReplay; }
 
+    void setPhysicalSQFullReplay() { setReplay(LdStReplayType::PhysicalSQFullReplay); }
+    bool needPhysicalSQFullReplay() const {
+        return getReplayType() == LdStReplayType::PhysicalSQFullReplay;
+    }
+
     void setSTLFReplay() { setReplay(LdStReplayType::STLFReplay); }
     bool needSTLFReplay() const { return getReplayType() == LdStReplayType::STLFReplay; }
 

src/cpu/o3/fetch.cc

@@ -144,7 +144,8 @@ Fetch::Fetch(CPU *_cpu, const BaseO3CPUParams &params)
     smtLdstqHighWater = params.smtBorrowLdstqHighWater;
     if (smtLdstqHighWater == 0) {
         smtLdstqHighWater =
-            (params.LQEntries + params.SQEntries) *
+            (params.LQEntries +
+             params.SQEntries * params.StoreQueueMultiple) *
             params.smtBorrowLdstqHighWaterPercent / 100;
     }
 

src/cpu/o3/inst_queue.cc

@@ -119,6 +119,13 @@ InstructionQueue::MdpAddrReplayLdInst::MdpAddrReplayLdInst(
 {
 }
 
+bool
+InstructionQueue::PhysicalSQFullReplayOrder::operator()(
+    const DynInstPtr &lhs, const DynInstPtr &rhs) const
+{
+    return lhs->seqNum > rhs->seqNum;
+}
+
 bool
 InstructionQueue::hasMdpAddrReplayInsts() const
 {
@@ -397,6 +404,9 @@ InstructionQueue::resetState()
     for (auto &replay_ld_insts : mdpAddrReplayLdInsts) {
         replay_ld_insts.clear();
     }
+    for (auto &replay_q : physicalSQFullReplayQs) {
+        replay_q = PhysicalSQFullReplayQueue();
+    }
     blockedMemInsts.clear();
     retryMemInsts.clear();
     wbOutstanding = 0;
@@ -434,6 +444,7 @@ InstructionQueue::isDrained() const
     bool drained = scheduler->isDrained() &&
                    instsToExecute.empty() &&
                    wbOutstanding == 0;
+    drained = drained && !hasPhysicalSQFullReplayInsts();
     for (ThreadID tid = 0; tid < numThreads; ++tid)
         drained = drained && memDepUnit[tid].isDrained();
 
@@ -444,6 +455,7 @@ void
 InstructionQueue::drainSanityCheck() const
 {
     assert(instsToExecute.empty());
+    assert(!hasPhysicalSQFullReplayInsts());
     for (ThreadID tid = 0; tid < numThreads; ++tid)
         memDepUnit[tid].drainSanityCheck();
 }
@@ -472,6 +484,7 @@ InstructionQueue::insert(const DynInstPtr &new_inst, int disp_seq)
     }
     // Make sure the instruction is valid
     assert(new_inst);
+    new_inst->instQueue = this;
 
     DPRINTF(IQ, "Adding instruction [sn:%llu] PC %s to the IQ.\n",
             new_inst->seqNum, new_inst->pcState());
@@ -495,6 +508,7 @@ InstructionQueue::insertNonSpec(const DynInstPtr &new_inst)
     }
 
     assert(new_inst);
+    new_inst->instQueue = this;
 
     scheduler->insertNonSpec(new_inst);
     nonSpecInsts[new_inst->seqNum] = new_inst;
@@ -632,6 +646,10 @@ InstructionQueue::scheduleReadyInsts()
         mem_inst->issueQue->retryMem(mem_inst);
     }
 
+    while ((mem_inst = getPhysicalSQFullReplayInstToExecute())) {
+        mem_inst->issueQue->retryMem(mem_inst);
+    }
+
     // Have iterator to head of the list
     // While I haven't exceeded bandwidth or reached the end of the list,
     // Try to get a FU that can do what this op needs.
@@ -1108,6 +1126,68 @@ InstructionQueue::getBlockedMemInstToExecute()
     }
 }
 
+void
+InstructionQueue::deferPhysicalSQFullReplay(const DynInstPtr &inst)
+{
+    assert(inst);
+    assert(inst->needPhysicalSQFullReplay());
+    iewStage->ldstQueue.recordStoreQueueReplay(inst);
+    physicalSQFullReplayQs[inst->threadNumber].push(inst);
+    DPRINTF(Schedule,
+            "Physical SQ replay waits in dedicated IQ queue [sn:%llu] "
+            "sqIdx=%llu\n",
+            inst->seqNum, static_cast<unsigned long long>(inst->sqIdx));
+}
+
+bool
+InstructionQueue::storeQueueWriteReady(const DynInstPtr &inst) const
+{
+    return iewStage->ldstQueue.storeQueueWriteReady(inst);
+}
+
+void
+InstructionQueue::recordAddrOrDataReady(const DynInstPtr &inst)
+{
+    iewStage->ldstQueue.recordAddrOrDataReady(inst);
+}
+
+DynInstPtr
+InstructionQueue::getPhysicalSQFullReplayInstToExecute()
+{
+    for (ThreadID tid = 0; tid < MaxThreads; ++tid) {
+        auto &replay_q = physicalSQFullReplayQs[tid];
+        while (!replay_q.empty()) {
+            const auto inst = replay_q.top();
+            if (!inst || inst->isSquashed()) {
+                replay_q.pop();
+                continue;
+            }
+            if (!iewStage->ldstQueue.phySQFullReplayReady(inst)) {
+                break;
+            }
+
+            replay_q.pop();
+            DPRINTF(Schedule,
+                    "Physical SQ replay wakes from IQ queue [sn:%llu] "
+                    "sqIdx=%llu\n",
+                    inst->seqNum, static_cast<unsigned long long>(inst->sqIdx));
+            return inst;
+        }
+    }
+    return nullptr;
+}
+
+bool
+InstructionQueue::hasPhysicalSQFullReplayInsts() const
+{
+    for (const auto &replay_q : physicalSQFullReplayQs) {
+        if (!replay_q.empty()) {
+            return true;
+        }
+    }
+    return false;
+}
+
 void
 InstructionQueue::violation(const DynInstPtr &store,
         const DynInstPtr &faulting_load)

src/cpu/o3/inst_queue.hh

@@ -226,6 +226,15 @@ class InstructionQueue
      */
     DynInstPtr getBlockedMemInstToExecute();
 
+    /** Insert a failed STA/STD into the physical-SQ replay wait queue. */
+    void deferPhysicalSQFullReplay(const DynInstPtr &inst);
+
+    /** Whether an STA/STD may write its current physical SQ window. */
+    bool storeQueueWriteReady(const DynInstPtr &inst) const;
+
+    /** Record the first address/data-ready transition for an SQ entry. */
+    void recordAddrOrDataReady(const DynInstPtr &inst);
+
     /** Process FU completion event. */
     void processFUCompletion(const DynInstPtr &inst, int fu_idx);
 
@@ -380,6 +389,20 @@ class InstructionQueue
 
     bool hasMdpAddrReplayInsts() const;
 
+    struct PhysicalSQFullReplayOrder
+    {
+        bool operator()(const DynInstPtr &lhs, const DynInstPtr &rhs) const;
+    };
+
+    using PhysicalSQFullReplayQueue = std::priority_queue<
+        DynInstPtr, std::vector<DynInstPtr>, PhysicalSQFullReplayOrder>;
+
+    std::array<PhysicalSQFullReplayQueue, MaxThreads>
+        physicalSQFullReplayQs;
+
+    DynInstPtr getPhysicalSQFullReplayInstToExecute();
+    bool hasPhysicalSQFullReplayInsts() const;
+
     /** List of instructions that have been cache blocked. */
     std::list<DynInstPtr> blockedMemInsts;
 

src/cpu/o3/issue_queue.cc

@@ -627,10 +627,17 @@ IssueQue::issueToFu()
 
     bool incTagRefillBlockStats = false;
 
-    // replay first
     for (; !replayQ.empty() && replayed < outports; replayed++) {
         auto& inst = replayQ.front();
 
+        // Do not let replay requests starve the older issuing requests
+        if (issueHasOlderInsts(inst)) {
+            DPRINTF(Schedule,
+                    "replay [sn:%llu] detected an older issuing request, delay this replay.\n",
+                    inst->seqNum);
+            break;
+        }
+
         if (inst->isLoad()) {
             // Loads selected here enter loadpipe S0 next cycle, so block on
             // the mainpipe tag-write state predicted for that admission point.
@@ -669,12 +676,16 @@ IssueQue::issueToFu()
             incTagRefillBlockStats = true;
         }
 
-        if ((issued >= outports) || (inst->isLoad() && (issuedLoad >= numLoadPipe)) ||
-            (inst->isStore() && (issuedStore >= numStorePipe)) || blockLoad) {
+        const bool issueOccupied =
+            (issued >= outports) ||
+            (inst->isLoad() && (issuedLoad >= numLoadPipe)) ||
+            (inst->isStore() && (issuedStore >= numStorePipe)) || blockLoad;
+        if (issueOccupied) {
             inst->clearScheduled();
             // only for load/store
             READYQ_PUSH(inst);
-            DPRINTF(Schedule, "[sn:%llu] issue failed due to being occupied\n", inst->seqNum);
+            DPRINTF(Schedule, "[sn:%llu] issue failed due to being occupied\n",
+                    inst->seqNum);
             continue;
         }
         if (!checkScoreboard(inst)) {
@@ -729,6 +740,24 @@ IssueQue::retryMem(const DynInstPtr& inst)
     replayQ.push(inst);
 }
 
+bool
+IssueQue::issueHasOlderInsts(const DynInstPtr& replay_inst) const
+{
+    if (!replay_inst) {
+        return false;
+    }
+
+    for (int i = 0; i < toFu->size; ++i) {
+        const auto &selected = toFu->insts[i];
+        if (selected &&
+            selected->threadNumber == replay_inst->threadNumber &&
+            selected->seqNum < replay_inst->seqNum) {
+            return true;
+        }
+    }
+    return false;
+}
+
 bool
 IssueQue::idle()
 {