cpu-o3: Fix SMT privilege-return fetch redirects

Change-Id: I03b2ac110f55736f2075a86ade085daa4893c4d1

Author: jensen-yan

src/cpu/o3/commit.cc

@@ -1866,6 +1866,8 @@ Commit::commitHead(const DynInstPtr &head_inst, unsigned inst_num)
     if (head_inst->isSerializeAfter() && head_inst->isNonSpeculative() &&
         head_inst->isReturn()) {
         traceLogPrivReturn(head_inst, tid);
+        // Restart fetch after the committed privilege state takes effect.
+        squashAfter(tid, head_inst);
     }
 
     committedPC[tid] = head_inst->pcState().instAddr();

src/cpu/o3/fetch.cc

@@ -557,13 +557,6 @@ Fetch::handleMultiCacheLineFetch(Addr vaddr, ThreadID tid, Addr pc)
 
     threads[tid].cacheReq.addRequest(first_mem_req); // packet will be created later
 
-    // Initiate translation for first request
-    updateCacheRequestStatusByRequest(tid, first_mem_req, TlbWait);
-    setAllFetchStalls(StallReason::ITlbStall);
-    FetchTranslation *trans = new FetchTranslation(this);
-    cpu->mmu->translateTiming(first_mem_req, cpu->thread[tid]->getTC(),
-                              trans, BaseMMU::Execute);
-
     // Prepare second request (head of second cache line)
     fetchPC += fetchSize;  // Move to start of next cache line
     assert(fetchPC % cacheBlkSize == 0);
@@ -584,6 +577,18 @@ Fetch::handleMultiCacheLineFetch(Addr vaddr, ThreadID tid, Addr pc)
 
     threads[tid].cacheReq.addRequest(second_mem_req);  // Add second request to cache request
 
+    // Initiate translations after both requests are registered. Some MMU paths
+    // complete synchronously, so callbacks must see the full request group.
+    updateCacheRequestStatusByRequest(tid, first_mem_req, TlbWait);
+    setAllFetchStalls(StallReason::ITlbStall);
+    FetchTranslation *trans = new FetchTranslation(this);
+    cpu->mmu->translateTiming(first_mem_req, cpu->thread[tid]->getTC(),
+                              trans, BaseMMU::Execute);
+
+    if (threads[tid].cacheReq.findRequestIndex(second_mem_req) == SIZE_MAX) {
+        return true;
+    }
+
     DPRINTF(Fetch, "[tid:%i] Initiating translation for second cache line\n", tid);
 
     // Always initiate translation for second request, regardless of first request status
@@ -950,17 +955,12 @@ Fetch::fetchCacheLine(Addr vaddr, ThreadID tid, Addr pc)
 bool
 Fetch::validateTranslationRequest(ThreadID tid, const RequestPtr &mem_req)
 {
-    // Check if this request belongs to current cache request
-    bool isExpectedReq = false;
-    for (size_t i = 0; i < threads[tid].cacheReq.requests.size(); i++) {
-        if (mem_req == threads[tid].cacheReq.requests[i]) {
-            isExpectedReq = true;
-            break;
-        }
-    }
+    const size_t reqIndex = threads[tid].cacheReq.findRequestIndex(mem_req);
 
-    // Check if request should be processed using new state system
-    if (!isExpectedReq || !hasPendingCacheRequests(tid)) {
+    // Check this request's status, not the request group's overall status:
+    // another cacheline in the same fetch may already have failed.
+    if (reqIndex == SIZE_MAX ||
+        threads[tid].cacheReq.requestStatus[reqIndex] != TlbWait) {
         DPRINTF(Fetch, "[tid:%i] Ignoring translation completed after squash or unexpected request\n", tid);
         DPRINTF(Fetch, "[tid:%i] Ignoring req addr=%#lx\n", tid, mem_req->getVaddr());
         ++fetchStats.tlbSquashes;

src/cpu/o3/iew.cc

@@ -1531,7 +1531,8 @@ IEW::SquashCheckAfterExe(DynInstPtr inst)
             inst->pcState(*new_pc);
         }
 
-        if (inst->mispredicted() && !loadNotExecuted) {
+        if (inst->mispredicted() && !loadNotExecuted &&
+            !inst->isNonSpeculative()) {
             fetchRedirect[tid] = true;
 
             DPRINTF(IEW, "[tid:%i] [sn:%llu] Execute: "
@@ -1978,7 +1979,7 @@ IEW::checkMisprediction(const DynInstPtr& inst)
             inst->pcState(*new_pc);
         }
 
-        if (inst->mispredicted()) {
+        if (inst->mispredicted() && !inst->isNonSpeculative()) {
             fetchRedirect[tid] = true;
 
             DPRINTF(IEW, "[tid:%i] [sn:%llu] Execute: "