feat(LoadQueueReplay): add support of fast wakeup for C_MA & C_FF

This Commit add support of fast wakeup from issueQueue of sta && std to LRQ.

We will send sqIdx at og0 stage of issueQueue, latch 1 cycle. We will use scoreBoard to trace the wakeup and detect wakeup cancel.

the source of wakeup cancel: og1Cancel, s0Cancel, s1Cancel.

Author: weidingliu

src/main/scala/xiangshan/Parameters.scala

@@ -106,6 +106,7 @@ case class XSCoreParameters
   LoadQueueRAWSize: Int = 56, // NOTE: make sure that LoadQueueRAWSize is power of 2.
   RollbackGroupSize: Int = 8,
   LoadQueueReplaySize: Int = 120,
+  LoadDependenceScoreBoardWidth: Int = 3,
   LoadUncacheBufferSize: Int = 16,
   LoadQueueNWriteBanks: Int = 8, // NOTE: make sure that LoadQueueRARSize/LoadQueueRAWSize is divided by LoadQueueNWriteBanks
   StoreQueueSize: Int = 64,
@@ -706,6 +707,7 @@ trait HasXSParameter {
   val RAWlgSelectGroupSize = log2Ceil(RollbackGroupSize)
   val RAWTotalDelayCycles = scala.math.ceil(log2Ceil(LoadQueueRAWSize).toFloat / RAWlgSelectGroupSize).toInt + 1 - 2
   def LoadQueueReplaySize = coreParams.LoadQueueReplaySize
+  def LoadDependenceScoreBoardWidth = coreParams.LoadDependenceScoreBoardWidth
   def LoadUncacheBufferSize = coreParams.LoadUncacheBufferSize
   def LoadQueueNWriteBanks = coreParams.LoadQueueNWriteBanks
   def StoreQueueSize = coreParams.StoreQueueSize

src/main/scala/xiangshan/XSCore.scala

@@ -198,6 +198,8 @@ class XSCoreImp(outer: XSCoreBase) extends LazyModuleImp(outer)
   memBlock.io.ooo_to_mem.backendToTopBypass := backend.io.toTop
   memBlock.io.ooo_to_mem.intIssue <> backend.io.mem.intIssue
   memBlock.io.ooo_to_mem.vecIssue <> backend.io.mem.vecIssue
+  memBlock.io.ooo_to_mem.wakeupToLRQ <> backend.io.mem.wakeupToLRQ
+  memBlock.io.ooo_to_mem.wakeupToLRQCancel := backend.io.mem.wakeupToLRQCancel
 
   // By default, instructions do not have exceptions when they enter the function units.
   memBlock.io.ooo_to_mem.intIssue.flatten.foreach { case x => x.bits.flushPipe.foreach(_ := false.B) }

src/main/scala/xiangshan/backend/Backend.scala

@@ -315,6 +315,8 @@ class BackendInlinedImp(override val wrapper: BackendInlined)(implicit p: Parame
   fpRegion.io.wakeupFromI2F. foreach(x => x := intRegion.io.cross.I2FWakeupOut.get)
   intRegion.io.F2IWakeupIn.  foreach(x => x := fpRegion.io.cross.F2IWakeupOut.get)
   intRegion.io.wakeupFromLDU.foreach(x => x := io.mem.wakeup)
+  intRegion.io.wakeupToLRQ.foreach(x => io.mem.wakeupToLRQ.zip(x.flatten).foreach { case (sink, source) => sink := source })
+  intRegion.io.wakeupToLRQCancel.foreach(x => io.mem.wakeupToLRQCancel.zip(x.flatten).foreach { case (sink, source) => sink := source })
   intRegion.io.staFeedback.  foreach(x => x := io.mem.staIqFeedback)
   vecRegion.io.vstuFeedback. foreach(x => x := io.mem.vstuIqFeedback)
   intRegion.io.ldCancel := io.mem.ldCancel
@@ -692,6 +694,8 @@ class BackendMemIO(implicit p: Parameters, params: BackendParams) extends XSBund
 
   val intIssue = intSchdParams.genExuInputCopySrcBundleMemBlock
   val vecIssue = vecSchdParams.genExuInputCopySrcBundleMemBlock
+  val wakeupToLRQ = Vec(params.StaCnt + params.StdCnt, ValidIO(new IssueQueueLRQWakeUpBundle))
+  val wakeupToLRQCancel = Vec(params.StaCnt + params.StdCnt, new IssueQueueLRQWakeUpCancelBundle)
 
   // store event difftest information
   val storeDebugInfo = Vec(EnsbufferWidth, new Bundle {

src/main/scala/xiangshan/backend/Bundles.scala

@@ -810,6 +810,14 @@ object Bundles {
     }
   }
 
+  class IssueQueueLRQWakeUpBundle(implicit p: Parameters) extends XSBundle {
+    val sqIdx = new SqPtr
+  }
+
+  class IssueQueueLRQWakeUpCancelBundle(implicit p: Parameters) extends XSBundle {
+    val og1Cancel = Bool()
+  }
+
   class VPUCtrlSignals(implicit p: Parameters) extends XSBundle {
     // vtype
     val vill      = Bool()
@@ -1378,7 +1386,7 @@ class ExuOutputVLoad(val params: ExeUnitParams)(implicit val p: Parameters) exte
     val perfDebugInfo  = Option.when(backendParams.debugEn)(new PerfDebugInfo())
     val debug_seqNum   = Option.when(backendParams.debugEn)(InstSeqNum())
   }
-  
+
   class MemDebugBundle(implicit p: Parameters) extends XSBundle {
     val isMMIO = Option.when(backendParams.basicDebugEn)(Bool())
     val isNCIO = Option.when(backendParams.basicDebugEn)(Bool())

src/main/scala/xiangshan/backend/Region.scala

@@ -183,6 +183,11 @@ class Region(val params: SchdBlockParams)(implicit p: Parameters) extends XSModu
     imp.io.s2Resp.get.head.lqIdx.foreach(_ := feedBack.bits.lqIdx)
     imp.io.s2Resp.get.head.sqIdx.foreach(_ := feedBack.bits.sqIdx)
   }
+  val stDataIQs = issueQueues.filter(iq => iq.param.StdCnt > 0)
+  if (params.isIntSchd) {
+    val lrqWakeupFromIQ = (stAddrIQs ++ stDataIQs).flatMap(_.io.wakeupToLRQ.get)
+    io.wakeupToLRQ.get.flatten.zip(lrqWakeupFromIQ).foreach { case (sink, source) => sink := source }
+  }
   val vecStuIQs = issueQueues.filter(iq => iq.param.VstuCnt > 0)
   vecStuIQs.zipWithIndex.foreach { case(imp, i) =>
     imp.io.memIO.get.lqDeqPtr.get := io.lqDeqPtr.get
@@ -258,7 +263,6 @@ class Region(val params: SchdBlockParams)(implicit p: Parameters) extends XSModu
     }
   }
   // std dispatch
-  val stDataIQs = issueQueues.filter(iq => iq.param.StdCnt > 0)
   val staEnqs = stAddrIQs.map(_.io.enq).flatten
   val stdEnqs = stDataIQs.map(_.io.enq).flatten.take(staEnqs.size)
   val noStdExuParams = params.issueBlockParams.map(x => Seq.fill(x.numEnq)(x.exuBlockParams)).flatten.filter { x => x.map(!_.hasStdFu).reduce(_ && _) }
@@ -483,6 +487,20 @@ class Region(val params: SchdBlockParams)(implicit p: Parameters) extends XSModu
     io.toMemExu.get <> toMem
     val firstMemExu = bypassNetwork.io.toExus.int.indexWhere(x => x.map(xx => xx.bits.params.isMemExeUnit).reduce(_ || _))
     println(s"[Regin_int] firstMemExu = $firstMemExu")
+    val firstStoreExu = bypassNetwork.io.toExus.int.indexWhere(x => x.map(xx => xx.bits.params.hasStoreFu).reduce(_ || _))
+    println(s"[Regin_int] firstStoreExu = $firstStoreExu")
+
+    val storeWakeupCancel = Wire(io.wakeupToLRQCancel.get.cloneType)
+
+    storeWakeupCancel.zipWithIndex.foreach { case (toMemCancel, i) =>
+      for (j <- toMemCancel.indices) {
+        val toMemExuInput = bypassNetwork.io.toExus.int(firstStoreExu + i)(j)
+        toMemCancel(j).og1Cancel := RegNext(LoadShouldCancel(toMemExuInput.bits.ctrl.loadDependency, io.ldCancel)) //regNext for timing
+      }
+    }
+
+    io.wakeupToLRQCancel.get.flatten.zip(storeWakeupCancel.flatten).foreach { case (sink, source) => sink := source }
+
     for (i <- toMem.indices) {
       for (j <- toMem(i).indices) {
         val toMemExuInput = bypassNetwork.io.toExus.int(firstMemExu + i)(j)
@@ -860,6 +878,9 @@ class RegionIO(val params: SchdBlockParams)(implicit p: Parameters) extends XSBu
   val wakeupFromF2I = Option.when(params.isIntSchd)(Flipped(ValidIO(new IssueQueueIQWakeUpBundle(params.backendParam.getExuIdxF2I, params.backendParam))))
   val cross = new ExuCrossRegion(params)
   val toMemExu = Option.when(!params.isFpSchd)(params.genNewExuInputCopySrcBundleMemBlock)
+  //to Mem, wake up LoadQueueReplay
+  val wakeupToLRQ = Option.when(params.isIntSchd)(intSchdParam.genMemWakeupLRQBundle)
+  val wakeupToLRQCancel = Option.when(params.isIntSchd)(intSchdParam.genMemWakeupCancelBundle)
   // fromMem
   val wakeupFromLDU = Option.when(params.isIntSchd)(Vec(params.LdExuCnt, Flipped(Valid(new MemWakeUpBundle))))
   val staFeedback = Option.when(params.isIntSchd)(Flipped(Vec(params.StaCnt, new MemRSFeedbackIO)))
@@ -945,4 +966,3 @@ class RegionIO(val params: SchdBlockParams)(implicit p: Parameters) extends XSBu
   val debugIQEnqHasIssuedVec = Option.when(backendParams.debugEn)(Vec(IQNum, Output(Bool())))
   val debugIQDeqRobIdxVec = Option.when(backendParams.debugEn)(Vec(iqDeqSum, ValidIO(new RobPtr())))
 }
-

src/main/scala/xiangshan/backend/issue/IssueBlockParams.scala

@@ -59,6 +59,8 @@ case class IssueBlockParams(
 
   def isStdIQ: Boolean = StdCnt > 0
 
+  def isStoreIQ: Boolean = isStAddrIQ || isStdIQ
+
   def isHyAddrIQ: Boolean = HyuCnt > 0
 
   def isVecLduIQ: Boolean = (VlduCnt + VseglduCnt) > 0
@@ -406,6 +408,14 @@ case class IssueBlockParams(
     MixedVec(this.exuBlockParams.map(x => DecoupledIO(x.genNewExuInputCopySrcBundle)))
   }
 
+  def genMemWakeupLRQBundle(implicit p: Parameters): MixedVec[ValidIO[IssueQueueLRQWakeUpBundle]] = {
+    MixedVec(this.exuBlockParams.filter(_.hasStoreFu).map(x => ValidIO(new IssueQueueLRQWakeUpBundle)))
+  }
+
+  def genMemWakeupCancelBundle(implicit p: Parameters): MixedVec[IssueQueueLRQWakeUpCancelBundle] = {
+    MixedVec(this.exuBlockParams.filter(x => x.hasStoreFu).map(x => new IssueQueueLRQWakeUpCancelBundle))
+  }
+
   def genExuOutputDecoupledBundle(implicit p: Parameters): MixedVec[DecoupledIO[ExuOutput]] = {
     MixedVec(this.exuParams.map(x => DecoupledIO(x.genExuOutputBundle)))
   }
@@ -487,6 +497,14 @@ case class IssueBlockParams(
     MixedVec(this.wakeUpInExuSources.map(x => ValidIO(new IssueQueueIQWakeUpBundle(backendParam.getExuIdx(x.name), backendParam))))
   }
 
+  def genIOWakeUpLRQValidBundle(implicit p: Parameters): MixedVec[ValidIO[IssueQueueLRQWakeUpBundle]] = {
+    MixedVec(exuBlockParams.filter(_.hasStoreFu).map(x => ValidIO(new IssueQueueLRQWakeUpBundle)))
+  }
+
+  def genIOWakeUpCancelBundle(implicit p: Parameters): MixedVec[IssueQueueLRQWakeUpCancelBundle] = {
+    MixedVec(exuBlockParams.filter(_.hasStoreFu).map(x => Output(new IssueQueueLRQWakeUpCancelBundle)))
+  }
+
   def genOGRespBundle(implicit p: Parameters) = {
     implicit val issueBlockParams = this
     MixedVec(exuBlockParams.map(_ => new OGRespBundle))

src/main/scala/xiangshan/backend/issue/IssueQueue.scala

@@ -34,6 +34,8 @@ class IssueQueueIO()(implicit p: Parameters, params: IssueBlockParams) extends X
   val wakeupFromF2I: Option[ValidIO[IssueQueueIQWakeUpBundle]] = Option.when(params.needWakeupFromF2I)(Flipped(ValidIO(new IssueQueueIQWakeUpBundle(params.backendParam.getExuIdxF2I, params.backendParam))))
   val wakeupFromWBDelayed: MixedVec[ValidIO[IssueQueueWBWakeUpBundle]] = Flipped(params.genWBWakeUpSinkValidBundle)
   val wakeupFromIQDelayed: MixedVec[ValidIO[IssueQueueIQWakeUpBundle]] = Flipped(params.genIQWakeUpSinkValidBundle)
+  //to Mem, wake up LoadQueueReplay
+  val wakeupToLRQ = Option.when(params.isStAddrIQ || params.isStdIQ)(params.genIOWakeUpLRQValidBundle)
   val vlFromIntIsZero = Input(Bool())
   val vlFromIntIsVlmax = Input(Bool())
   val vlFromVfIsZero = Input(Bool())
@@ -948,6 +950,16 @@ class IssueQueueImp(implicit p: Parameters, params: IssueBlockParams) extends XS
     dontTouch(io.deqDelay)
     dontTouch(deqBeforeDly)
   }
+  // sta wake up LRQ in og1
+  io.wakeupToLRQ.foreach { case wakeupOption =>
+    wakeupOption.zipWithIndex.foreach { case (wakeup, i) =>
+      val wakeupCancel = LoadShouldCancel(io.deqDelay(i).bits.loadDependency, io.ldCancel)
+      val wakeupValid = io.deqDelay(i).fire && !wakeupCancel
+      wakeup.valid := RegNext(wakeupValid) // next cycle is og1
+      wakeup.bits.sqIdx := RegNext(entries.io.deqOg1Payload(i).sqIdx.get)
+    }
+  }
+
   io.wakeupToIQ.zipWithIndex.foreach { case (wakeup, i) =>
     if (wakeUpQueues(i).nonEmpty) {
       wakeup.valid := wakeUpQueues(i).get.io.deq.valid

src/main/scala/xiangshan/backend/issue/SchdBlockParams.scala

@@ -103,7 +103,7 @@ case class SchdBlockParams(
   def needSrcVxrm: Boolean = issueBlockParams.map(_.needSrcVxrm).reduce(_ || _)
 
   def writeVConfig: Boolean = issueBlockParams.map(_.writeVConfig).reduce(_ || _)
-  
+
   def writeVType: Boolean = issueBlockParams.map(_.writeVType).reduce(_ || _)
 
   def numRedirect: Int = issueBlockParams.map(_.numRedirect).sum
@@ -152,6 +152,14 @@ case class SchdBlockParams(
     MixedVec(this.issueBlockParams.filter(_.isMemBlockIQ).map(_.genNewExuInputDecoupledCopySrcBundle))
   }
 
+  def genMemWakeupLRQBundle(implicit p: Parameters): MixedVec[MixedVec[ValidIO[IssueQueueLRQWakeUpBundle]]] = {
+    MixedVec(this.issueBlockParams.filter(_.isStoreIQ).map(_.genMemWakeupLRQBundle))
+  }
+
+  def genMemWakeupCancelBundle(implicit p: Parameters): MixedVec[MixedVec[IssueQueueLRQWakeUpCancelBundle]] = {
+    MixedVec(this.issueBlockParams.filter(_.isStoreIQ).map(_.genMemWakeupCancelBundle))
+  }
+
   def genExuOutputDecoupledBundle(implicit p: Parameters): MixedVec[MixedVec[DecoupledIO[ExuOutput]]] = {
     MixedVec(this.issueBlockParams.map(_.genExuOutputDecoupledBundle))
   }
@@ -187,7 +195,7 @@ case class SchdBlockParams(
   def genExuOutputDecoupledBundleNoMemBlock(implicit p: Parameters): MixedVec[MixedVec[DecoupledIO[ExuOutput]]] = {
     MixedVec(this.issueBlockParams.filterNot(_.isMemBlockIQ).map(_.genExuOutputDecoupledBundle))
   }
-  
+
   def genNewExuOutputDecoupledBundleNoMemBlock(implicit p: Parameters): MixedVec[MixedVec[DecoupledIO[NewExuOutput]]] = {
     MixedVec(this.issueBlockParams.filterNot(_.isMemBlockIQ).map(_.genNewExuOutputDecoupledBundle))
   }

src/main/scala/xiangshan/mem/MemBlock.scala

@@ -194,6 +194,8 @@ class ooo_to_mem(implicit p: Parameters) extends MemBlockBundle {
 
   val intIssue: MixedVec[MixedVec[DecoupledIO[ExuInput]]] = Flipped(intSchdParams.genExuInputCopySrcBundleMemBlock)
   val vecIssue: MixedVec[MixedVec[DecoupledIO[ExuInput]]] = Flipped(vecSchdParams.genExuInputCopySrcBundleMemBlock)
+  val wakeupToLRQ = Flipped(Vec(StaCnt + StdCnt, ValidIO(new IssueQueueLRQWakeUpBundle)))
+  val wakeupToLRQCancel = Input(Vec(StaCnt + StdCnt, new IssueQueueLRQWakeUpCancelBundle))
 }
 
 class mem_to_ooo(implicit p: Parameters) extends MemBlockBundle {
@@ -439,6 +441,7 @@ class MemBlockInlinedImp(outer: MemBlockInlined) extends LazyModuleImp(outer)
 
   val intIssue: Seq[DecoupledIO[ExuInput]] = io.ooo_to_mem.intIssue.flatten
   val vecIssue: Seq[DecoupledIO[ExuInput]] = io.ooo_to_mem.vecIssue.flatten
+  val wakeupToLRQCancel = Wire(Vec(StaCnt + StdCnt, new LRQWakeUpCancelBundle))
   val issueLda = intIssue.filter(_.bits.params.hasLoadFu)
   val issueSta = intIssue.filter(_.bits.params.hasStoreAddrFu)
   val issueStd = intIssue.filter(_.bits.params.hasStdFu)
@@ -492,6 +495,19 @@ class MemBlockInlinedImp(outer: MemBlockInlined) extends LazyModuleImp(outer)
   val newLoadUnits = Seq.tabulate(LduCnt)(i => Module(new NewLoadUnit(ldaParams(i))))
   val storeUnits = Seq.tabulate(StaCnt)(i => Module(new NewStoreUnit(staParams(i))))
   val stdExeUnits = Seq.tabulate(StdCnt)(i => Module(new StdExeUnit(stdParams(i))))
+
+  wakeupToLRQCancel.take(StaCnt).zip(storeUnits).zip(io.ooo_to_mem.wakeupToLRQCancel.take(StaCnt)).foreach {
+    case ((sink, stu), source) =>
+      sink.og1Cancel := source.og1Cancel //had latch in issueQueue
+      sink.s0Cancel := RegNext(!stu.io.stin.ready)
+      sink.s1Cancel := stu.io.feedBackSlow.valid && !stu.io.feedBackSlow.bits.hit // had latch in stu, so no need to latch again here
+  }
+  wakeupToLRQCancel.drop(StaCnt).take(StdCnt).zip(stdExeUnits).zip(io.ooo_to_mem.wakeupToLRQCancel.drop(StaCnt).take(StdCnt)).foreach {
+    case ((sink, std), source) =>
+      sink.og1Cancel := source.og1Cancel //had latch in issueQueue
+      sink.s0Cancel := RegNext(!std.io.in.ready)
+      sink.s1Cancel := false.B // TODO: used in the future
+  }
   val atomicsUnit = Module(new AtomicsUnit(mouParam))
 
   // The number of vector load/store units is decoupled with the number of load/store units
@@ -1028,6 +1044,8 @@ class MemBlockInlinedImp(outer: MemBlockInlined) extends LazyModuleImp(outer)
 
   //  lsq.io.rob            <> io.lsqio.rob
   lsq.io.enq            <> io.ooo_to_mem.enqLsq
+  lsq.io.wakeupToLRQ <> io.ooo_to_mem.wakeupToLRQ
+  lsq.io.wakeupToLRQCancel := wakeupToLRQCancel
   lsq.io.brqRedirect    <> redirect
 
   //  violation rollback

src/main/scala/xiangshan/mem/lsqueue/LSQBundle.scala

@@ -28,6 +28,12 @@ import xiangshan.cache.{CMOReq, CMOResp, DCacheWordReqWithVaddrAndPfFlag, Uncach
 import xiangshan.frontend.ftq.FtqPtr
 import xiangshan.mem.Bundles.{SQForward, StoreMaskBundle}
 
+  class LRQWakeUpCancelBundle(implicit p: Parameters) extends XSBundle {
+    val og1Cancel = Bool()
+    val s0Cancel  = Bool()
+    val s1Cancel  = Bool()
+  }
+
 class StoreQueueEnqIO(implicit p: Parameters) extends MemBlockBundle {
   // Bundle define
 
@@ -112,7 +118,7 @@ class StoreAddrIO(implicit p: Parameters) extends MemBlockBundle {
   * means this write request need to write whole cacheline.
   * */
   val wlineflag          = Bool() // store write the whole cache line.
-  
+
   // misalign
   val isUnalign   = Bool()
   val cross16Byte = Bool()