#!/usr/bin/env python3
"""LuLu expiration watchdog.
Works around a LuLu bug (all versions since 2.9.1, incl. 4.4.0 prerelease):
-[Rules load] deletes expired "temporarily allowed" rules while enumerating
them -> uncaught NSException -> the network extension crash-loops at startup
-> firewall is silently OFF until rules.plist is repaired by hand.
This watchdog performs the deletion LuLu itself intended, but safely: any
rule whose expiration date has already passed is REMOVED from the rules file
(preserving temporary-rule semantics — LuLu will simply re-prompt on the
app's next connection). If an app's last rule is removed, the whole app
entry is removed, matching LuLu's own delete behavior. Rules whose
expiration is still in the future are left untouched (the extension handles
those safely at runtime).
It also restarts the extension whenever it is down and its last exit was a
crash (macOS only auto-relaunches a couple of times right at boot and then
gives up; launching the LuLu app manually just crashes again). This covers
both the boot race (crash before our repair landed) and an already-throttled
provider. A deliberately stopped extension (clean exit / not registered) is
never touched. Restart attempts are rate-limited so a crash with a DIFFERENT
cause can't make us kick forever. `--kick` forces a restart check regardless
(manual recovery, e.g. instead of a reboot).
After a successful revive it relaunches LuLu.app if it was already running:
the app does not re-establish its XPC link to a revived provider, so alerts
would stay silent — and per XPCListener.m, unmatched flows are ALLOWED while
no client is connected. At boot the app isn't running yet (it launches after
login and connects fresh), so this only triggers on runtime revives.
Intended to run as a root LaunchDaemon (RunAtLoad + StartInterval + WatchPaths).
Usage: lulu-expiration-watchdog.py [--kick] [path-to-rules.plist]
Exit code is always 0 unless the patched file could not be written safely.
"""
import datetime
import os
import plistlib
import re
import shutil
import subprocess
import sys
import tempfile
import time
ARGS = sys.argv[1:]
MANUAL_KICK = '--kick' in ARGS
PATH_ARGS = [a for a in ARGS if a != '--kick']
RULES = PATH_ARGS[0] if PATH_ARGS else '/Library/Objective-See/LuLu/rules.plist'
CUSTOM_PATH = bool(PATH_ARGS) # test mode: never touch launchd
APPLE_EPOCH = datetime.datetime(2001, 1, 1, tzinfo=datetime.timezone.utc)
EXT_PROC = 'com.objective-see.lulu.extension'
KICK_MARKER = '/var/run/lulu-watchdog.kicks'
KICK_LIMIT = 3 # max automatic kicks ...
KICK_WINDOW = 3600 # ... per hour
def log(msg):
print(f'lulu-watchdog: {msg}', flush=True)
def extension_running():
return subprocess.run(['/usr/bin/pgrep', '-x', EXT_PROC],
capture_output=True).returncode == 0
def find_extension_service():
"""Discover the versioned launchd job and its last exit status.
Returns (label, last_exit) — last_exit < 0 means killed by a signal
(a crash, e.g. -6 = SIGABRT), None if unknown/never ran.
"""
out = subprocess.run(['/bin/launchctl', 'print', 'system'],
capture_output=True, text=True).stdout
m = re.search(r'^\s*\S+\s+(-?\d+)\s+'
r'(NetworkExtension\.com\.objective-see\.lulu\.extension\.\S+)\s*$',
out, flags=re.M)
if m:
return m.group(2), int(m.group(1))
m = re.search(r'(NetworkExtension\.com\.objective-see\.lulu\.extension\.\S+)', out)
return (m.group(1), None) if m else (None, None)
def kick_allowed():
"""Rate-limit automatic kicks: at most KICK_LIMIT per KICK_WINDOW."""
try:
st = os.stat(KICK_MARKER)
count = int(open(KICK_MARKER).read().strip() or 0)
if time.time() - st.st_mtime > KICK_WINDOW:
count = 0
if count >= KICK_LIMIT:
return False, count
return True, count
except FileNotFoundError:
return True, 0
except Exception:
return True, 0
def record_kick(count):
try:
with open(KICK_MARKER, 'w') as f:
f.write(str(count + 1))
except Exception:
pass
def console_user():
"""uid of the logged-in GUI user, or None (login window)."""
try:
uid = os.stat('/dev/console').st_uid
return uid if uid != 0 else None
except Exception:
return None
def relaunch_lulu_app():
"""A revived provider is invisible to an already-running LuLu.app (stale
XPC link: alerts silently undelivered, unmatched flows allowed) — so
relaunch the app to make it reconnect. No-op if nobody is logged in or
the app isn't running (it will connect fresh when it launches)."""
uid = console_user()
if uid is None:
return
if subprocess.run(['/usr/bin/pgrep', '-x', 'LuLu'],
capture_output=True).returncode != 0:
return
subprocess.run(['/usr/bin/pkill', '-x', 'LuLu'])
time.sleep(1)
r = subprocess.run(['/bin/launchctl', 'asuser', str(uid),
'/usr/bin/open', '-a', 'LuLu'],
capture_output=True, text=True)
if r.returncode == 0:
log('relaunched LuLu.app so it reconnects to the revived extension '
'(its stale XPC link would silently drop alerts otherwise)')
else:
log(f'could not relaunch LuLu.app (rc={r.returncode}: {r.stderr.strip()}) '
f'— relaunch it manually so alerts reconnect')
def proc_elapsed(name):
"""Seconds the named process has been running, or None if not running.
macOS ps has no Linux-style `etimes`, so parse `etime` ([[dd-]hh:]mm:ss)."""
r = subprocess.run(['/usr/bin/pgrep', '-x', name],
capture_output=True, text=True)
if r.returncode != 0:
return None
pid = r.stdout.split()[0]
r = subprocess.run(['/bin/ps', '-o', 'etime=', '-p', pid],
capture_output=True, text=True)
try:
s = r.stdout.strip()
days = 0
if '-' in s:
d, s = s.split('-', 1)
days = int(d)
parts = [int(x) for x in s.split(':')]
while len(parts) < 3:
parts.insert(0, 0)
return days * 86400 + parts[0] * 3600 + parts[1] * 60 + parts[2]
except (ValueError, AttributeError, IndexError):
return None
def heal_stale_app_link():
"""If LuLu.app has been running LONGER than the extension, the extension
was restarted underneath it (by us, or by macOS's own crash respawn) and
the app's XPC link is stale — relaunch the app. Self-limiting: after the
relaunch the app is younger than the extension, so this never repeats."""
if CUSTOM_PATH:
return
ext, app = proc_elapsed(EXT_PROC), proc_elapsed('LuLu')
if ext is None or app is None:
return
if app > ext + 30: # 30s grace against boot-time ordering jitter
log(f'LuLu.app (up {app}s) predates the extension (up {ext}s) — '
f'stale XPC link, alerts would be silently dropped; relaunching the app')
relaunch_lulu_app()
def kick_extension(manual, repaired):
"""Restart the extension if it is down and crashed (or on manual request)."""
if CUSTOM_PATH:
return
if extension_running():
if manual:
log('extension already running — nothing to kick')
return
svc, last_exit = find_extension_service()
if svc is None:
if manual:
log('could not find extension launchd service — is LuLu installed/enabled?')
return
if not manual and not repaired and (last_exit is None or last_exit >= 0):
return # down but not crashed -> deliberately stopped; leave it alone
if not manual:
ok, count = kick_allowed()
if not ok:
log(f'extension down but kick limit reached ({count}/{KICK_LIMIT} per hour) — not kicking')
return
record_kick(count)
log(f'extension is down (last exit {last_exit}) — restarting it')
r = subprocess.run(['/bin/launchctl', 'kickstart', f'system/{svc}'],
capture_output=True, text=True)
if r.returncode == 0:
time.sleep(2)
revived = extension_running()
log(f'kickstarted {svc} — extension now {"RUNNING" if revived else "still down"}')
if revived:
relaunch_lulu_app()
return
else:
log(f'kickstart {svc} failed (rc={r.returncode}): {r.stderr.strip()}')
if manual:
# last resort, manual only: bounce the NE session manager so it
# re-establishes all enabled providers (may blip VPN sessions)
r = subprocess.run(['/bin/launchctl', 'kickstart', '-k',
'system/com.apple.nesessionmanager'],
capture_output=True, text=True)
time.sleep(3)
revived = extension_running()
log(f'bounced nesessionmanager (rc={r.returncode}) — extension now '
f'{"RUNNING" if revived else "still down"}')
if revived:
relaunch_lulu_app()
def main():
if not os.path.exists(RULES):
return 0 # LuLu not installed / no rules yet — nothing to do
try:
stat_before = os.stat(RULES)
with open(RULES, 'rb') as f:
raw = f.read()
fmt = plistlib.FMT_BINARY if raw[:8] == b'bplist00' else plistlib.FMT_XML
d = plistlib.loads(raw)
objs = d['$objects']
root = objs[d['$top']['root'].data] # outer NSMutableDictionary
if not (isinstance(root, dict) and 'NS.keys' in root and 'NS.objects' in root):
raise ValueError('unexpected root object')
except Exception as e: # unreadable/corrupt — never touch it, just report
log(f'ERROR: cannot parse {RULES}: {e}')
return 0
now = datetime.datetime.now(datetime.timezone.utc)
# pass 1: collect expired rule objects (never mutate while scanning)
expired = [] # (rule_obj_index, name, iso_expiry)
try:
for i, o in enumerate(objs):
if not (isinstance(o, dict) and isinstance(o.get('expiration'), plistlib.UID)):
continue
uid = o['expiration'].data
if uid == 0:
continue # permanent rule
tgt = objs[uid]
if not (isinstance(tgt, dict) and 'NS.time' in tgt):
continue
expires = APPLE_EPOCH + datetime.timedelta(seconds=tgt['NS.time'])
if expires <= now:
name = objs[o['name'].data] if isinstance(o.get('name'), plistlib.UID) else '?'
expired.append((i, str(name), expires.isoformat()))
except Exception as e:
log(f'ERROR: unexpected structure in {RULES}: {e}')
return 0
if not expired:
# nothing to repair, but still revive a crashed extension and
# reconnect an app whose provider was restarted underneath it
kick_extension(manual=MANUAL_KICK, repaired=False)
heal_stale_app_link()
return 0 # healthy — stay silent
# pass 2: delete each expired rule the way LuLu's own delete: would —
# drop its reference from the per-app rules array; if that array becomes
# empty, drop the whole app entry from the root dictionary.
# ($objects entries become unreferenced orphans, which NSKeyedUnarchiver
# ignores — indices never shift, so all other UIDs stay valid.)
try:
prune_root = set()
for ridx, name, expires in expired:
for ai, arr in enumerate(objs):
# per-app rules arrays are the only NS.objects-lists that
# reference Rule objects (sets hold strings, dicts have NS.keys)
if not (isinstance(arr, dict) and 'NS.objects' in arr and 'NS.keys' not in arr):
continue
if not any(isinstance(u, plistlib.UID) and u.data == ridx for u in arr['NS.objects']):
continue
arr['NS.objects'] = [u for u in arr['NS.objects'] if u.data != ridx]
if not arr['NS.objects']:
for i, inner_uid in enumerate(root['NS.objects']):
inner = objs[inner_uid.data]
if (isinstance(inner, dict)
and any(isinstance(v, plistlib.UID) and v.data == ai
for v in inner.get('NS.objects', []))):
prune_root.add(i)
break
break
for i in sorted(prune_root, reverse=True):
del root['NS.keys'][i]
del root['NS.objects'][i]
except Exception as e:
log(f'ERROR: failed to remove expired rule(s): {e}')
return 0
backup = RULES + '.wdbackup-' + now.strftime('%Y%m%d-%H%M%S')
try:
shutil.copy2(RULES, backup)
# write atomically next to the original, bail if LuLu wrote meanwhile
fd, tmp = tempfile.mkstemp(prefix='.rules-wd-', dir=os.path.dirname(RULES))
try:
with os.fdopen(fd, 'wb') as f:
plistlib.dump(d, f, fmt=fmt)
os.chmod(tmp, 0o644)
stat_now = os.stat(RULES)
if (stat_now.st_mtime_ns, stat_now.st_size) != (stat_before.st_mtime_ns, stat_before.st_size):
log('file changed while patching (extension is alive?) — skipping, next run will retry')
os.unlink(tmp)
os.unlink(backup)
return 0
os.replace(tmp, RULES)
except Exception:
if os.path.exists(tmp):
os.unlink(tmp)
raise
except Exception as e:
log(f'ERROR: failed to write patched rules: {e}')
return 1
for i, name, expires in expired:
log(f'deleted expired rule {name!r} (obj#{i}, expired {expires}) — LuLu will re-prompt on next connect')
log(f'backup: {backup}')
# the file is clean now — if the extension already burned its automatic
# relaunch attempts on the broken file, restart it ourselves
kick_extension(manual=MANUAL_KICK, repaired=True)
heal_stale_app_link()
return 0
if __name__ == '__main__':
sys.exit(main())
Environment
cd3be6c("new feature: rule expiration", 2024-08-25) already contains the delete-during-enumeration. It first appeared in the v2.9.1 prerelease (Sep 2024) and went stable with v3.0.0 (Jan 2025); I checked theloadmethod of every tagged release since — the faulty pattern is present in all of them, through v4.3.2, the 4.4.0 prerelease, and current master (that block's only change in two years is one added comment line). So effectively every build of the last ~2 years is affectedSymptoms
EXC_CRASH (SIGABRT).systemextensionsctl list:[activated enabled]), so the advice leads nowhere: toggling the extension off/on in System Settings changes nothing, and the same alert returns on every app launch, because the provider crashes instantly no matter how its start is triggered./Library/Logs/DiagnosticReports/fills up withcom.objective-see.lulu.extension-*.ips, all identical.Security impact
I'd argue this is more than a stability bug: normal use of a built-in feature ("temporarily allow", no attacker involved) puts the firewall into a silent, persistent fail-open state:
Root cause
-[Rules load](LuLu/Extension/Rules.m) deletes expired rules while fast-enumerating the very collections it mutates:Foundation throws
NSGenericException— "*** Collection <…> was mutated while being enumerated" (__NSFastEnumerationMutationHandler) — the exception is uncaught, the runtime callsabort().Since the crash happens before the updated rules are persisted, the expired rule stays in
rules.plist, so every subsequent start crashes again: a permanent crash-loop that survives reboots and app re-installs. This also explains why installing the latest beta doesn't help — the trigger lives in the on-disk rules file.Both enumerations can trap; in my crash reports the faulting PC alternates between
load+1140 andload+1324 (4.4.0, arm64 slice), i.e. inner vs. outer loop, consistent across dozens of reports.Crash report excerpt
(procLaunch → captureTime: 37 ms. Happy to share full
.ipsfiles privately.)Steps to reproduce
dispatch_afterpath deletes the rule safely instead.Verified on my machine:
rules.plistcontained exactly one rule with an expiration date (expired the previous evening); every extension launch since then crashed within ~40 ms.Fix verified: after removing the expired rule from
rules.plist, the extension came straight back — no reboot needed, revived vialaunchctl kickstarton its launchd job — and has been running stably with zero further crash reports.Suggested fix
Collect expired rules during enumeration and delete after the loops — the same
rules2Deletepattern-[Rules cleanup:]already uses:(A defensive "purge expired rules" pass before the fixup loops in
loadwould also self-heal existing users stuck in the crash-loop.)Workaround for affected users
Unbrick the extension: null out (or remove) the expired rule's
expirationin/Library/Objective-See/LuLu/rules.plist(root-owned, NSKeyedArchiver format — set the rule dict'sexpirationUID to 0/$null). Then restart the provider: reboot, or without a reboot kick its launchd job directly —sudo launchctl kickstart "system/$(launchctl print system | grep -o 'NetworkExtension\.com\.objective-see\.lulu\.extension\.[^ ]*')"(this is what the daemon below automates). Deletingrules.plistalso works but loses all rules.Keep using the feature safely until a fix ships: I now run a small root LaunchDaemon that checks the rules file on boot, every 5 minutes, and on every change to the file, and performs the deletion
-[Rules load]itself intended: any already-expired rule is removed from the file (if it was the app's last rule, the whole app entry is removed, matching LuLu's owndelete:behavior) — so temporary-rule semantics are preserved and LuLu simply re-prompts on the app's next connection. Future expirations are left alone since the extension's runtime timer path handles those correctly. The daemon also restarts the extension vialaunchctl kickstartwhenever it is down and its last exit was a crash (rate-limited; a deliberately stopped extension — clean exit — is never touched). Since macOS only auto-relaunches a couple of times right at boot, this is what actually keeps the firewall up;--kickdoes the same on demand, which recovers a throttled provider without a reboot.One more pitfall the daemon handles: an already-running LuLu.app does not re-establish its XPC link to a revived provider — alerts stay silent, and per
XPCListener.munmatched flows are allowed while no client is connected, so the silence is even fail-open. This bites after ANY provider restart underneath a running app (a kickstart, macOS's own crash respawn, or a System-Settings toggle). The daemon therefore relaunches the app after a revive, and additionally watches for the tell-tale inversion "app has been running longer than the extension" to heal stale links from any cause. (Might be worth a fix in the app too: re-attach when the provider restarts.) Script and daemon plist below.lulu-expiration-watchdog.py (install to
/usr/local/bin/, mode 755, owner root:wheel)/Library/LaunchDaemons/local.lulu-expiration-watchdog.plist (mode 644, owner root:wheel)
Load with:
sudo launchctl bootstrap system /Library/LaunchDaemons/local.lulu-expiration-watchdog.plistNote: run the
sudo installfrom a non-TCC-protected source location (e.g./tmpor/Users/Shared) — macOS blocks even root from reading~/Desktop/~/Documents/~/Downloadsunless the terminal has Full Disk Access.