timing(CoupledL2): optimize timing

src/main/scala/coupledL2/Common.scala

@@ -262,6 +262,8 @@ class MSHRInfo(implicit p: Parameters) extends L2Bundle with HasTLChannelBits {
   val mergeA = Bool() // whether the mshr already merge an acquire(avoid alias merge)
 
   val w_grantfirst = Bool()
+  val w_grantlast = Bool()
+  val w_grant = Bool()
   val s_release = Bool()
   val s_refill = Bool()
   val s_cmoresp = Bool()
@@ -399,8 +401,8 @@ class PrefetchRecv extends Bundle {
 }
 
 // custom l2 - l1 interface
-class L2ToL1Hint(implicit p: Parameters) extends Bundle {
-  val sourceId = UInt(32.W)    // tilelink sourceID
+class L2ToL1Hint(implicit p: Parameters) extends L2Bundle {
+  val sourceId = UInt(sourceIdBits.W)    // tilelink sourceID
   val isKeyword = Bool()       // miss entry keyword
 }
 

src/main/scala/coupledL2/CoupledL2.scala

@@ -34,6 +34,7 @@ import coupledL2.prefetch._
 import huancun.{BankBitsKey, TPmetaReq, TPmetaResp}
 import utility.mbist.{MbistInterface, MbistPipeline}
 import utility.sram.{SramBroadcastBundle, SramHelper}
+import coupledL2.utils._
 
 trait HasCoupledL2Parameters {
   val p: Parameters
@@ -211,18 +212,28 @@ trait HasCoupledL2Parameters {
     x(x.getWidth - 1, pageOffsetBits)
   }
 
-  def arb[T <: Bundle](in: Seq[DecoupledIO[T]], out: DecoupledIO[T], name: Option[String] = None): Unit = {
+  def arb[T <: Bundle](in: Seq[DecoupledIO[T]], out: DecoupledIO[T], name: Option[String] = None) = {
     val arb = Module(new Arbiter[T](chiselTypeOf(out.bits), in.size))
     if (name.nonEmpty) { arb.suggestName(s"${name.get}_arb") }
     for ((a, req) <- arb.io.in.zip(in)) { a <> req }
     out <> arb.io.out
+    arb
   }
 
-  def fastArb[T <: Bundle](in: Seq[DecoupledIO[T]], out: DecoupledIO[T], name: Option[String] = None): Unit = {
+  def fastArb[T <: Bundle](in: Seq[DecoupledIO[T]], out: DecoupledIO[T], name: Option[String] = None) = {
     val arb = Module(new FastArbiter[T](chiselTypeOf(out.bits), in.size))
     if (name.nonEmpty) { arb.suggestName(s"${name.get}_arb") }
     for ((a, req) <- arb.io.in.zip(in)) { a <> req }
     out <> arb.io.out
+    arb
+  }
+
+  def twoLevelArb[T <: Bundle](in: Seq[DecoupledIO[T]], out: DecoupledIO[T], name: Option[String] = None) = {
+    val arb = Module(new TwoLevelRRArbiter(chiselTypeOf(out.bits), in.size))
+    if (name.nonEmpty) { arb.suggestName(s"${name.get}_arb") }
+    for ((a, req) <- arb.io.in.zip(in)) { a <> req }
+    out <> arb.io.out
+    arb
   }
 
   def odOpGen(r: UInt) = {
@@ -327,7 +338,7 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
       val hartId = Input(UInt(hartIdLen.W))
       val pfCtrlFromCore = Input(new PrefetchCtrlFromCore)
     //  val l2_hint = Valid(UInt(32.W))
-      val l2_hint = ValidIO(new L2ToL1Hint())
+      val l2_hint = ValidIO(new L2ToL1Hint()(l2ECCParams))
       val l2_tlb_req = new L2ToL1TlbIO(nRespDups = 1)(l2TlbParams)
       val debugTopDown = new Bundle {
         val robTrueCommit = Input(UInt(64.W))
@@ -526,7 +537,7 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
     if (enableHintGuidedGrant) {
       // for timing consideration, hint should latch one cycle before sending to L1
       // instead of adding a Pipeline/Queue to latch here, we just set hintQueue in GrantBuf & CustomL1Hint "flow=false"
-      val l1HintArb = Module(new Arbiter(new L2ToL1Hint(), slices.size))
+      val l1HintArb = Module(new Arbiter(new L2ToL1Hint()(l2ECCParams), slices.size))
       val slices_l1Hint = slices.zipWithIndex.map {
         case (s, i) => s.io.l1Hint
       }

src/main/scala/coupledL2/CustomL1Hint.scala

@@ -26,13 +26,14 @@ import coupledL2.utils._
 
 class HintQueueEntry(implicit p: Parameters) extends L2Bundle {
   val source = UInt(sourceIdBits.W)
-  val opcode = UInt(3.W)
+  val isGrantData = Bool()
   val isKeyword = Bool()
 }
 
 class CustomL1HintIOBundle(implicit p: Parameters) extends L2Bundle {
   // input information
-  val s1 = Flipped(ValidIO(new TaskBundle()))
+  val mshrHintQInfo = Flipped(ValidIO(new TaskBundle()))
+  val sinkCHintQInfo = Flipped(ValidIO(new TaskBundle()))
   val s3 = new L2Bundle {
       val task      = Flipped(ValidIO(new TaskBundle()))
       val need_mshr = Input(Bool())
@@ -47,76 +48,69 @@ class CustomL1HintIOBundle(implicit p: Parameters) extends L2Bundle {
 class CustomL1Hint(implicit p: Parameters) extends L2Module {
   val io = IO(new CustomL1HintIOBundle)
 
-  val task_s1 = io.s1
+  val mshr_s1 = io.mshrHintQInfo.bits
+  val mshrMerge_s1 = mshr_s1.aMergeTask
+  val sinkC_s1 = io.sinkCHintQInfo.bits
   val task_s3 = io.s3.task
-  val mshrReq_s1 = task_s1.bits.mshrTask
   val mshrReq_s3 = task_s3.bits.mshrTask
-  val mergeA_s1  = task_s1.bits.mergeA
   val need_mshr_s3 = io.s3.need_mshr
 
-  def isGrantData(t: TaskBundle):  Bool = t.fromA && t.opcode === GrantData
-  def isGrant(t: TaskBundle):      Bool = t.fromA && t.opcode === Grant
-  def isHintAck(t: TaskBundle):    Bool = t.fromA && t.opcode === HintAck // HintAck has no effect on Hint
-  def isRelease(t: TaskBundle):    Bool = t.fromC && (t.opcode === Release || t.opcode === ReleaseData)
-  def isMergeGrantData(t: TaskBundle): Bool = t.fromA && t.mergeA && t.aMergeTask.opcode === GrantData
-  def isMergeGrant(t: TaskBundle):     Bool = t.fromA && t.mergeA && t.aMergeTask.opcode === Grant
-
   // ==================== Hint Generation ====================
   // Hint for "MSHRTask and ReleaseAck" will fire@s1
-  val mshr_GrantData_s1 = task_s1.valid &&  mshrReq_s1 && (isGrantData(task_s1.bits) || isMergeGrantData(task_s1.bits))
-  val mshr_Grant_s1     = task_s1.valid &&  mshrReq_s1 && (isGrant(task_s1.bits) || isMergeGrant(task_s1.bits))
-  val chn_Release_s1    = task_s1.valid && !mshrReq_s1 && isRelease(task_s1.bits)
+  // val mshr_GrantData_s1 = task_s1.valid &&  mshrReq_s1 && (isGrantData(task_s1.bits) || isMergeGrantData(task_s1.bits))
+  // val mshr_Grant_s1     = task_s1.valid &&  mshrReq_s1 && (isGrant(task_s1.bits) || isMergeGrant(task_s1.bits))
+  // val chn_Release_s1    = task_s1.valid && !mshrReq_s1 && isRelease(task_s1.bits)
+  val mshr_GrantData_s1 = io.mshrHintQInfo.valid && (mshr_s1.fromA && (mshr_s1.opcode === GrantData || (mshr_s1.mergeA && mshrMerge_s1.opcode === GrantData)))
+  val mshr_Grant_s1     = io.mshrHintQInfo.valid && (mshr_s1.fromA && (mshr_s1.opcode === Grant || (mshr_s1.mergeA && mshrMerge_s1.opcode === Grant)))
+  val chn_Release_s1    = io.sinkCHintQInfo.valid
+  assert(Mux(chn_Release_s1, sinkC_s1.fromC, true.B))
+  assert(Mux(chn_Release_s1, sinkC_s1.opcode === Release || sinkC_s1.opcode === ReleaseData, true.B))
 
-  val enqValid_s1 = mshr_GrantData_s1 || mshr_Grant_s1 || chn_Release_s1
-  val enqSource_s1 = Mux(task_s1.bits.mergeA, task_s1.bits.aMergeTask.sourceId, task_s1.bits.sourceId)
-  val enqKeyWord_s1 = Mux(task_s1.bits.mergeA,
-    task_s1.bits.aMergeTask.isKeyword.getOrElse(false.B),
-    task_s1.bits.isKeyword.getOrElse(false.B)
-  )
-  val enqOpcode_s1 = ParallelPriorityMux(
-    Seq(
-      mshr_Grant_s1 -> Grant,
-      mshr_GrantData_s1 -> GrantData,
-      chn_Release_s1 -> ReleaseAck
-    )
-  )
+  val enqBits_s1 = Wire(new HintQueueEntry)
+  // enqBits_s1.source := Mux(task_s1.bits.mergeA, task_s1.bits.aMergeTask.sourceId, task_s1.bits.sourceId)
+  enqBits_s1.source := Mux1H(Seq(
+    (io.mshrHintQInfo.valid && mshr_s1.mergeA) -> mshrMerge_s1.sourceId,
+    (io.mshrHintQInfo.valid && !mshr_s1.mergeA) -> mshr_s1.sourceId,
+    io.sinkCHintQInfo.valid -> sinkC_s1.sourceId
+  ))
+  OneHot.checkOneHot(Cat(io.mshrHintQInfo.valid && mshr_s1.mergeA, io.mshrHintQInfo.valid && !mshr_s1.mergeA, io.sinkCHintQInfo.valid))
+  enqBits_s1.isKeyword := Mux(mshr_s1.mergeA, mshrMerge_s1.isKeyword.getOrElse(false.B), mshr_s1.isKeyword.getOrElse(false.B)) 
+  enqBits_s1.isGrantData := mshr_GrantData_s1
 
   // Hint for "chnTask Hit" will fire@s3
-  val chn_Grant_s3     = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && isGrant(task_s3.bits)
-  val chn_GrantData_s3 = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && isGrantData(task_s3.bits)
+  val chn_Grant_s3     = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && task_s3.bits.fromA && task_s3.bits.opcode === Grant
+  val chn_GrantData_s3 = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && task_s3.bits.fromA && task_s3.bits.opcode === GrantData
+  val enqBits_s3 = Wire(new HintQueueEntry)
   val enqValid_s3 = chn_Grant_s3 || chn_GrantData_s3
-  val enqSource_s3 = task_s3.bits.sourceId
-  val enqKeyWord_s3 = task_s3.bits.isKeyword.getOrElse(false.B)
-  val enqOpcode_s3 = ParallelPriorityMux(
-    Seq(
-      chn_Grant_s3 -> Grant,
-      chn_GrantData_s3 -> GrantData
-    )
-  )
+  enqBits_s3.source := task_s3.bits.sourceId
+  enqBits_s3.isKeyword := task_s3.bits.isKeyword.getOrElse(false.B)
+  enqBits_s3.isGrantData := chn_GrantData_s3
 
   // ==================== Hint Queue ====================
   val hintEntries = mshrsAll
   val hintEntriesWidth = log2Ceil(hintEntries)
   val hintQueue = Module(new Queue(new HintQueueEntry, hintEntries))
+  val canFlow_s1 = !hintQueue.io.deq.valid || hintQueue.io.count === 1.U && hintQueue.io.deq.fire
+  val valid_s1 = mshr_GrantData_s1 || mshr_Grant_s1 || chn_Release_s1
+  val flow_s1, enq_s3 = Wire(Decoupled(new HintQueueEntry))
+  // noSpaceForSinkReq in GrantBuffer may ensure that these queues will not overflow
+  assert(enq_s3.ready || !enq_s3.valid)
 
   // this will have at most 2 entries
-  val hint_s1Queue = Module(new Queue(new HintQueueEntry, 4, flow = true))
-  hint_s1Queue.io.enq.valid := enqValid_s1
-  hint_s1Queue.io.enq.bits.opcode := enqOpcode_s1
-  hint_s1Queue.io.enq.bits.source := enqSource_s1
-  hint_s1Queue.io.enq.bits.isKeyword := enqKeyWord_s1
-  hint_s1Queue.io.deq.ready := hintQueue.io.enq.ready && !enqValid_s3
-  // WARNING:TODO: ensure queue will never overflow
-  assert(hint_s1Queue.io.enq.ready, "hint_s1Queue should never be full")
-  assert(hintQueue.io.enq.ready, "hintQueue should never be full")
+  val hint_s1Queue = Module(new Pipeline(new HintQueueEntry))
+  hint_s1Queue.io.in.valid := valid_s1 && (!canFlow_s1 || !flow_s1.ready)
+  hint_s1Queue.io.in.bits  := enqBits_s1
+  assert(!valid_s1 || hint_s1Queue.io.in.ready || flow_s1.ready)
+
+  flow_s1.valid := valid_s1 && canFlow_s1
+  flow_s1.bits := enqBits_s1
 
-  hintQueue.io.enq.valid := enqValid_s3 || hint_s1Queue.io.deq.valid
-  hintQueue.io.enq.bits.opcode := Mux(enqValid_s3, enqOpcode_s3, hint_s1Queue.io.deq.bits.opcode)
-  hintQueue.io.enq.bits.source := Mux(enqValid_s3, enqSource_s3, hint_s1Queue.io.deq.bits.source)
-  hintQueue.io.enq.bits.isKeyword := Mux(enqValid_s3, enqKeyWord_s3, hint_s1Queue.io.deq.bits.isKeyword)
+  enq_s3.valid := enqValid_s3
+  enq_s3.bits := enqBits_s3
+  arb(Seq(enq_s3,  hint_s1Queue.io.out, flow_s1), hintQueue.io.enq, Some("Hint"))
   hintQueue.io.deq.ready := io.l1Hint.ready
 
-  io.l1Hint.valid := hintQueue.io.deq.valid && hintQueue.io.deq.bits.opcode === GrantData
+  io.l1Hint.valid := hintQueue.io.deq.valid && hintQueue.io.deq.bits.isGrantData
   io.l1Hint.bits.sourceId := hintQueue.io.deq.bits.source
   io.l1Hint.bits.isKeyword := hintQueue.io.deq.bits.isKeyword
 }

src/main/scala/coupledL2/DataStorage.scala

@@ -27,6 +27,7 @@ class DSRequest(implicit p: Parameters) extends L2Bundle {
   val way = UInt(wayBits.W)
   val set = UInt(setBits.W)
   val wen = Bool()
+  val ren = Bool()
 }
 
 // mask not used
@@ -78,12 +79,12 @@ class DataStorage(implicit p: Parameters) extends L2Module {
     extraHold = true,
     withClockGate = true
   ))
-  array.io_en := io.en
+  array.io_en := io.en && (io.req.bits.ren || io.req.bits.wen)
   private val mbistPl = MbistPipeline.PlaceMbistPipeline(1, "L2DataStorage", p(L2ParamKey).hasMbist)
 
   val arrayIdx = Cat(io.req.bits.way, io.req.bits.set)
   val wen = io.req.valid && io.req.bits.wen
-  val ren = io.req.valid && !io.req.bits.wen
+  val ren = io.req.valid && !io.req.bits.wen && io.req.bits.ren
 
   val arrayWrite = Wire(new DSECCBankBlock)
   val arrayWriteData = if (enableDataECC) {
@@ -121,12 +122,12 @@ class DataStorage(implicit p: Parameters) extends L2Module {
   io.rdata := dataRead
   io.error := error
 
-  assert(!io.en || !RegNext(io.en, false.B),
+  assert(!array.io_en || !RegNext(array.io_en, false.B),
     "Continuous SRAM req prohibited under MCP2!")
 
-  assert(!(RegNext(io.en) && (io.req.asUInt =/= RegNext(io.req.asUInt))),
+  assert(!(RegNext(array.io_en) && (io.req.asUInt =/= RegNext(io.req.asUInt))),
     s"DataStorage req fails to hold for 2 cycles!")
 
-  assert(!(RegNext(io.en && io.req.bits.wen) && (io.wdata.asUInt =/= RegNext(io.wdata.asUInt))),
+  assert(!(RegNext(array.io_en && io.req.bits.wen) && (io.wdata.asUInt =/= RegNext(io.wdata.asUInt))),
     s"DataStorage wdata fails to hold for 2 cycles!")
 }