Merge pull request #265 from plasma-umass/lief_port

macOS port: LIEF-based binary parsing and causal profiling

Author: emeryberger

.github/workflows/ci.yml

@@ -35,50 +35,49 @@ jobs:
         cmake .. -DCMAKE_BUILD_TYPE=RelWithDebInfo
         make -j$(nproc)
 
-    - name: Build toy benchmark
+    - name: Build benchmarks
       run: |
         cd build
         cmake .. -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo
-        make -j$(nproc) toy
+        make -j$(nproc) toy lock_test
 
     - name: Verify build artifacts
       run: |
         test -f build/libcoz/libcoz.so
         test -x build/benchmarks/toy/toy
+        test -x build/benchmarks/lock_test/lock_test
         echo "Build artifacts verified"
 
     - name: Run toy benchmark with coz
       run: |
         cd build
         # Run coz with the toy benchmark for a short duration
-        timeout 30 ../coz run -o profile.coz --- ./benchmarks/toy/toy || true
+        timeout 30 ../coz run -o profile.jsonl --- ./benchmarks/toy/toy || true
 
-        # Check that profile was created
-        if [ ! -f profile.coz ]; then
-          echo "ERROR: profile.coz was not created"
+        if [ ! -f profile.jsonl ]; then
+          echo "ERROR: profile.jsonl was not created"
           exit 1
         fi
 
         echo "Profile created successfully"
 
-    - name: Validate profile output
+    - name: Validate toy profile output
       run: |
         cd build
 
-        # Check profile has content
-        if [ ! -s profile.coz ]; then
-          echo "ERROR: profile.coz is empty"
+        if [ ! -s profile.jsonl ]; then
+          echo "ERROR: profile.jsonl is empty"
           exit 1
         fi
 
         echo "=== Profile Contents ==="
-        cat profile.coz
+        cat profile.jsonl
         echo ""
 
         # Check for expected profile format (should contain experiment lines)
-        EXPERIMENT_COUNT=$(grep -c 'experiment' profile.coz || echo 0)
+        EXPERIMENT_COUNT=$(grep -c 'experiment' profile.jsonl || echo 0)
         echo "=== Profile Summary ==="
-        echo "  Total lines: $(wc -l < profile.coz)"
+        echo "  Total lines: $(wc -l < profile.jsonl)"
         echo "  Experiments: $EXPERIMENT_COUNT"
 
         if [ "$EXPERIMENT_COUNT" -eq 0 ]; then
@@ -87,8 +86,7 @@ jobs:
         fi
 
         # Validate profile references toy.cpp line 18 (the loop in function a())
-        # This is the critical line that should show optimization potential
-        if grep -q "toy.cpp:18" profile.coz; then
+        if grep -q "toy.cpp:18" profile.jsonl; then
           echo "  Found experiments for toy.cpp:18 (loop in function a())"
         else
           echo "ERROR: toy.cpp:18 not found in profile - expected loop in a() to be profiled"
@@ -98,44 +96,27 @@ jobs:
         echo ""
         echo "Profile validation PASSED"
 
-    - name: Validate optimization potential
+    - name: Validate toy optimization potential
       run: |
         cd build
 
-        # Validate that toy.cpp:18 shows expected optimization behavior:
-        # - Linear speedup potential up to ~50% (because a() takes 2x as long as b())
-        # - Diminishing returns beyond 50% (speeding up a() more doesn't help since b() becomes bottleneck)
-
         python3 << 'VALIDATION_SCRIPT'
         import sys
-        import re
+        import json
 
-        # Parse profile.coz
-        # Format: experiment\tselected=path:line\tspeedup=X.XX\tduration=N\tselected-samples=N
-        # Followed by: throughput-point\tname=path:line\tdelta=N
         experiments = []
         current_exp = None
 
-        with open('profile.coz', 'r') as f:
+        with open('profile.jsonl', 'r') as f:
             for line in f:
                 line = line.strip()
-                if line.startswith('experiment'):
-                    # Parse key=value pairs
-                    parts = line.split('\t')
-                    exp = {}
-                    for part in parts[1:]:
-                        if '=' in part:
-                            key, val = part.split('=', 1)
-                            exp[key] = val
-                    current_exp = exp
-                elif line.startswith('throughput-point') and current_exp:
-                    # Parse throughput point and attach to current experiment
-                    parts = line.split('\t')
-                    for part in parts[1:]:
-                        if '=' in part:
-                            key, val = part.split('=', 1)
-                            if key == 'delta':
-                                current_exp['delta'] = float(val)
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if obj.get('type') == 'experiment':
+                    current_exp = obj
+                elif obj.get('type') == 'throughput_point' and current_exp:
+                    current_exp['delta'] = float(obj['delta'])
                     experiments.append(current_exp)
                     current_exp = None
 
@@ -151,7 +132,7 @@ jobs:
         # Group by speedup percentage and calculate average delta
         speedup_deltas = {}
         for e in line18_exps:
-            s = int(float(e['speedup']) * 100)  # Convert decimal (0.40) to percentage (40)
+            s = int(float(e['speedup']) * 100)
             if s not in speedup_deltas:
                 speedup_deltas[s] = []
             speedup_deltas[s].append(e['delta'])
@@ -162,7 +143,6 @@ jobs:
             avg_delta = sum(deltas) / len(deltas)
             print(f"  {speedup}%: avg_delta={avg_delta:.2f} (n={len(deltas)})")
 
-        # Validation: Check that we have data for multiple speedup levels
         if len(speedup_deltas) < 3:
             print(f"WARNING: Only {len(speedup_deltas)} speedup levels - need more data for accurate validation")
         else:
@@ -172,9 +152,80 @@ jobs:
         print("\nProfile validation COMPLETE")
         VALIDATION_SCRIPT
 
-    - name: Upload profile artifact
+    - name: Run lock_test benchmark with coz
+      run: |
+        cd build
+        timeout 60 ../coz run -o lock_test_profile.jsonl --- ./benchmarks/lock_test/lock_test || true
+
+        if [ ! -f lock_test_profile.jsonl ]; then
+          echo "ERROR: lock_test_profile.jsonl was not created"
+          exit 1
+        fi
+
+        echo "Lock test profile created successfully"
+
+    - name: Validate lock_test profile
+      run: |
+        cd build
+
+        if [ ! -s lock_test_profile.jsonl ]; then
+          echo "ERROR: lock_test_profile.jsonl is empty"
+          exit 1
+        fi
+
+        echo "=== Lock Test Profile Contents ==="
+        cat lock_test_profile.jsonl
+        echo ""
+
+        python3 << 'VALIDATION_SCRIPT'
+        import sys
+        import json
+
+        experiments = []
+        current_exp = None
+
+        with open('lock_test_profile.jsonl', 'r') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if obj.get('type') == 'experiment':
+                    current_exp = obj
+                elif obj.get('type') == 'throughput_point' and current_exp:
+                    current_exp['delta'] = float(obj['delta'])
+                    experiments.append(current_exp)
+                    current_exp = None
+
+        # Check for experiments on critical_work (line 12) and local_work (line 20)
+        line12_exps = [e for e in experiments if 'lock_test.cpp:12' in e['selected']]
+        line20_exps = [e for e in experiments if 'lock_test.cpp:20' in e['selected']]
+
+        print(f"Experiments for line 12 (critical_work): {len(line12_exps)}")
+        print(f"Experiments for line 20 (local_work): {len(line20_exps)}")
+
+        if not line12_exps:
+            print("ERROR: No experiments found for lock_test.cpp:12 (critical section)")
+            sys.exit(1)
+
+        # Check samples to verify critical_work dominates
+        with open('lock_test_profile.jsonl', 'r') as f:
+            for line in f:
+                obj = json.loads(line.strip())
+                if obj.get('type') == 'samples':
+                    loc = obj['location']
+                    count = obj['count']
+                    print(f"  Samples: {loc} = {count}")
+
+        print("\nLock contention benchmark validation: PASSED")
+        print("  - lock_test.cpp:12 (critical section) identified as profiling target")
+        VALIDATION_SCRIPT
+
+    - name: Upload profile artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: coz-profile
-        path: build/profile.coz
+        name: coz-profiles
+        path: |
+          build/profile.jsonl
+          build/lock_test_profile.jsonl
         retention-days: 7

CLAUDE.md

@@ -215,6 +215,22 @@ coz run --source-scope '/media/psf/Home/git/coz-portage/benchmarks/**' --- ./ben
 COZ_FILTER_SYSTEM=1 coz run --- ./benchmarks/toy/toy
 ```
 
+**Verbose output:**
+
+Use `--verbose` (or `-v`) to see what libraries and source files coz is processing:
+
+```bash
+coz run --verbose --- ./myapp
+```
+
+This prints:
+- Bootstrap messages
+- MAIN executable path resolution
+- Source files found in DWARF debug info
+- Libraries being profiled
+
+Useful for debugging when coz isn't finding expected source lines or to verify which files are in scope.
+
 ### Adding Progress Points
 
 Include `include/coz.h` and add macros:

CMakeLists.txt

@@ -11,6 +11,48 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/cmake)
 
 find_package(Threads REQUIRED)
 
+# ============== LIEF library for cross-platform binary parsing ==============
+option(USE_SYSTEM_LIEF "Link against LIEF discovered via find_package" OFF)
+
+if(USE_SYSTEM_LIEF)
+    find_package(LIEF REQUIRED)
+else()
+    include(FetchContent)
+
+    set(LIEF_GIT_TAG "0.17.0" CACHE STRING
+        "Git tag/branch/commit for LIEF")
+    set(LIEF_GIT_REPO "https://github.com/lief-project/LIEF.git" CACHE STRING
+        "Git repository URL for LIEF")
+
+    # Disable LIEF features we don't need to speed up build
+    set(LIEF_PYTHON_API OFF CACHE BOOL "" FORCE)
+    set(LIEF_C_API OFF CACHE BOOL "" FORCE)
+    set(LIEF_EXAMPLES OFF CACHE BOOL "" FORCE)
+    set(LIEF_TESTS OFF CACHE BOOL "" FORCE)
+    set(LIEF_DOC OFF CACHE BOOL "" FORCE)
+    set(LIEF_LOGGING OFF CACHE BOOL "" FORCE)
+    set(LIEF_ENABLE_JSON OFF CACHE BOOL "" FORCE)
+    # Only enable formats we need
+    set(LIEF_ELF ON CACHE BOOL "" FORCE)
+    set(LIEF_PE OFF CACHE BOOL "" FORCE)
+    set(LIEF_MACHO ON CACHE BOOL "" FORCE)
+    set(LIEF_OAT OFF CACHE BOOL "" FORCE)
+    set(LIEF_DEX OFF CACHE BOOL "" FORCE)
+    set(LIEF_VDEX OFF CACHE BOOL "" FORCE)
+    set(LIEF_ART OFF CACHE BOOL "" FORCE)
+    set(LIEF_COFF OFF CACHE BOOL "" FORCE)
+
+    FetchContent_Declare(
+        LIEF
+        GIT_REPOSITORY ${LIEF_GIT_REPO}
+        GIT_TAG        ${LIEF_GIT_TAG}
+    )
+
+    FetchContent_MakeAvailable(LIEF)
+    message(STATUS "Fetched LIEF ${LIEF_GIT_TAG}")
+endif()
+
+# ============== libelfin for DWARF parsing ==============
 option(USE_SYSTEM_LIBELFIN "Link against libelfin discovered via pkg-config" OFF)
 
 if(USE_SYSTEM_LIBELFIN)
@@ -25,8 +67,8 @@ else()
         # Fetch libelfin from GitHub
         include(FetchContent)
 
-        set(LIBELFIN_GIT_TAG "8779775beea1ee5444ba53d14e354dbcf42a811d" CACHE STRING
-            "Git tag/branch/commit for libelfin (default: includes pre-built to_string.cc)")
+        set(LIBELFIN_GIT_TAG "origin/improved_dwarf5_support" CACHE STRING
+            "Git tag/branch/commit for libelfin (default: DWARF5 support with pre-built to_string.cc)")
         set(LIBELFIN_GIT_REPO "https://github.com/plasma-umass/libelfin.git" CACHE STRING
             "Git repository URL for libelfin")
 
@@ -86,8 +128,8 @@ option(INSTALL_COZ "Enable installation of coz. (Projects embedding coz may want
 if(INSTALL_COZ)
     include(GNUInstallDirs)
     include(CMakePackageConfigHelpers)
-    install(PROGRAMS coz DESTINATION bin)
-    install(FILES LICENSE.md DESTINATION licenses)
+    install(PROGRAMS coz DESTINATION ${CMAKE_INSTALL_BINDIR})
+    install(FILES LICENSE.md DESTINATION ${CMAKE_INSTALL_DOCDIR})
 
     # Install the viewer web application
     install(DIRECTORY viewer/

benchmarks/lock_test/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_executable(lock_test lock_test.cpp)
+target_link_libraries(lock_test PRIVATE pthread coz-instrumentation)
+
+add_coz_run_target(run_lock_test COMMAND $<TARGET_FILE:lock_test>)

benchmarks/lock_test/lock_test.cpp

@@ -0,0 +1,62 @@
+#include <coz.h>
+#include <pthread.h>
+#include <stdio.h>
+
+// Shared mutex — the bottleneck
+static pthread_mutex_t the_lock = PTHREAD_MUTEX_INITIALIZER;
+
+// Work inside the critical section (the bottleneck)
+static volatile unsigned long long shared_counter;
+
+void critical_work() {
+  for (volatile int i = 0; i < 5000000; i++) {
+    shared_counter++;  // line 14 — should be the bottleneck
+  }
+}
+
+// Work outside the critical section (not the bottleneck)
+void local_work() {
+  volatile unsigned long long x = 0;
+  for (volatile int i = 0; i < 1000000; i++) {
+    x++;  // line 22 — should show ~0% impact
+  }
+}
+
+struct thread_arg {
+  int iterations;
+};
+
+void* worker(void* arg) {
+  int iters = ((thread_arg*)arg)->iterations;
+  for (int i = 0; i < iters; i++) {
+    local_work();
+
+    pthread_mutex_lock(&the_lock);
+    critical_work();
+    pthread_mutex_unlock(&the_lock);
+
+    COZ_PROGRESS;
+  }
+  return nullptr;
+}
+
+int main() {
+  const int NUM_THREADS = 4;
+  const int ITERS_PER_THREAD = 500;
+
+  printf("Lock contention test: %d threads, %d iterations each\n", NUM_THREADS, ITERS_PER_THREAD);
+
+  pthread_t threads[NUM_THREADS];
+  thread_arg args[NUM_THREADS];
+
+  for (int i = 0; i < NUM_THREADS; i++) {
+    args[i].iterations = ITERS_PER_THREAD;
+    pthread_create(&threads[i], nullptr, worker, &args[i]);
+  }
+
+  for (int i = 0; i < NUM_THREADS; i++) {
+    pthread_join(threads[i], nullptr);
+  }
+
+  printf("Done.\n");
+}

benchmarks/toy/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_executable(toy toy.cpp)
-target_link_libraries(toy PRIVATE pthread)
+target_link_libraries(toy PRIVATE pthread coz-instrumentation)
 
 add_coz_run_target(run_toy COMMAND $<TARGET_FILE:toy>)