Skip to content

Commit ff0152c

Browse files
pouyanafisipouyanafisi
committed
Add deterministic filtering for analytics, tracking, and source maps
- Create exclude-patterns.js with comprehensive patterns for: - Source maps (.js.map, .css.map) - Well-known paths (/.well-known/) - Analytics (Google, Facebook, Hotjar, Mixpanel, etc.) - Error tracking (Sentry, LogRocket, etc.) - Chat widgets (Intercom, Drift, etc.) - Ad networks and cookie consent - API endpoints that won't work offline - Strip analytics scripts from captured HTML - Remove tracking pixels and beacon links - Add --keep-analytics flag for users who need them
1 parent d0eae68 commit ff0152c

6 files changed

Lines changed: 464 additions & 1 deletion

File tree

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "smippo",
3-
"version": "0.1.2",
3+
"version": "0.1.3",
44
"description": "S.M.I.P.P.O. — Structured Mirroring of Internet Pages and Public Objects. Modern website copier that captures sites exactly as they appear in your browser.",
55
"main": "src/index.js",
66
"bin": {

‎src/cli.js‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,10 @@ export function run() {
157157
.option('--pdf', 'Save PDF of each page')
158158
.option('--static', 'Remove scripts for static offline viewing')
159159
.option('--inline-css', 'Inline CSS into HTML for single-file output')
160+
.option(
161+
'--keep-analytics',
162+
'Keep analytics/tracking scripts (stripped by default)',
163+
)
160164

161165
// Performance options
162166
.option('-w, --workers <n>', 'Parallel workers/pages (default: 8)', '8')
@@ -388,6 +392,7 @@ async function capture(url, options) {
388392
pdf: options.pdf,
389393
noJs: options.static,
390394
inlineCss: options.inlineCss,
395+
keepAnalytics: options.keepAnalytics,
391396
concurrency: parseInt(options.workers || options.concurrency, 10),
392397
maxPages: options.maxPages ? parseInt(options.maxPages, 10) : undefined,
393398
maxTime: options.maxTime ? parseInt(options.maxTime, 10) * 1000 : undefined,

‎src/crawler.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ export class Crawler extends EventEmitter {
310310
structure: this.options.structure,
311311
noJs: this.options.noJs,
312312
inlineCss: this.options.inlineCss,
313+
keepAnalytics: this.options.keepAnalytics,
313314
});
314315

315316
// Save HTML

‎src/filters/exclude-patterns.js‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
/**
2+
* Patterns for resources that should be excluded from capture
3+
* These are typically analytics, tracking, source maps, and other
4+
* resources that are not needed for offline viewing
5+
*/
6+
7+
/**
8+
* URL patterns to exclude from asset collection
9+
* These patterns are matched against the full URL
10+
*/
11+
export const EXCLUDE_URL_PATTERNS = [
12+
// Source maps (development files, not needed for viewing)
13+
/\.map$/i,
14+
/\.js\.map$/i,
15+
/\.css\.map$/i,
16+
17+
// Well-known paths (service discovery, not needed offline)
18+
/\/\.well-known\//i,
19+
20+
// Cloudflare
21+
/\/cdn-cgi\/rum/i, // Real User Monitoring
22+
/\/cdn-cgi\/beacon/i,
23+
/\/cdn-cgi\/trace/i,
24+
/\/cdn-cgi\/challenge-platform/i,
25+
/cloudflareinsights\.com/i,
26+
27+
// Google Analytics / Tag Manager
28+
/google-analytics\.com/i,
29+
/googletagmanager\.com/i,
30+
/googletagservices\.com/i,
31+
/googlesyndication\.com/i,
32+
/googleadservices\.com/i,
33+
/doubleclick\.net/i,
34+
/\/gtag\/js/i,
35+
/\/ga\.js/i,
36+
/\/analytics\.js/i,
37+
/\/gtm\.js/i,
38+
39+
// Facebook
40+
/connect\.facebook\.net/i,
41+
/facebook\.com\/tr/i,
42+
/fbevents\.js/i,
43+
/pixel\.facebook\.com/i,
44+
45+
// Twitter/X
46+
/platform\.twitter\.com\/widgets/i,
47+
/analytics\.twitter\.com/i,
48+
/t\.co\/i\/adsct/i,
49+
50+
// LinkedIn
51+
/snap\.licdn\.com/i,
52+
/linkedin\.com\/px/i,
53+
54+
// Microsoft/Bing
55+
/clarity\.ms/i,
56+
/bat\.bing\.com/i,
57+
58+
// Hotjar
59+
/hotjar\.com/i,
60+
/static\.hotjar\.com/i,
61+
62+
// Mixpanel
63+
/mixpanel\.com/i,
64+
/cdn\.mxpnl\.com/i,
65+
66+
// Segment
67+
/cdn\.segment\.com/i,
68+
/api\.segment\.io/i,
69+
70+
// Amplitude
71+
/amplitude\.com/i,
72+
/cdn\.amplitude\.com/i,
73+
74+
// Heap
75+
/heap-analytics\.com/i,
76+
/heapanalytics\.com/i,
77+
78+
// Intercom
79+
/widget\.intercom\.io/i,
80+
/api-iam\.intercom\.io/i,
81+
/intercom\.io\/widget/i,
82+
83+
// Drift
84+
/js\.driftt\.com/i,
85+
/drift\.com/i,
86+
87+
// HubSpot
88+
/js\.hs-scripts\.com/i,
89+
/js\.hsforms\.net/i,
90+
/js\.hs-analytics\.net/i,
91+
/forms\.hubspot\.com/i,
92+
/track\.hubspot\.com/i,
93+
94+
// Zendesk
95+
/static\.zdassets\.com/i,
96+
/ekr\.zdassets\.com/i,
97+
98+
// Crisp
99+
/client\.crisp\.chat/i,
100+
101+
// Tawk.to
102+
/embed\.tawk\.to/i,
103+
104+
// LiveChat
105+
/cdn\.livechatinc\.com/i,
106+
107+
// Sentry (error tracking)
108+
/browser\.sentry-cdn\.com/i,
109+
/sentry\.io\/api/i,
110+
/ingest\.sentry\.io/i,
111+
112+
// LogRocket
113+
/cdn\.logrocket\.io/i,
114+
/r\.logrocket\.io/i,
115+
116+
// FullStory
117+
/fullstory\.com/i,
118+
/rs\.fullstory\.com/i,
119+
120+
// Crazy Egg
121+
/script\.crazyegg\.com/i,
122+
123+
// Optimizely
124+
/cdn\.optimizely\.com/i,
125+
/logx\.optimizely\.com/i,
126+
127+
// VWO
128+
/dev\.visualwebsiteoptimizer\.com/i,
129+
130+
// New Relic
131+
/js-agent\.newrelic\.com/i,
132+
/bam\.nr-data\.net/i,
133+
134+
// Datadog
135+
/datadoghq\.com/i,
136+
/browser-intake/i,
137+
138+
// Beacon APIs (generic)
139+
/\/beacon\//i,
140+
/\/collect\?/i,
141+
/\/pixel\?/i,
142+
/\/track\?/i,
143+
/\/event\?/i,
144+
/\/log\?/i,
145+
146+
// Common API patterns that won't work offline
147+
/\/api\/v\d+\//i, // versioned APIs
148+
/graphql/i,
149+
/\/webhook/i,
150+
/\/socket\.io/i,
151+
/\/ws\//i, // WebSocket
152+
153+
// Social widgets (load external content)
154+
/platform\.instagram\.com/i,
155+
/apis\.google\.com\/js\/plusone/i,
156+
/widgets\.pinterest\.com/i,
157+
158+
// Ad networks
159+
/ad\.doubleclick\.net/i,
160+
/pagead2\.googlesyndication\.com/i,
161+
/adservice\.google\.com/i,
162+
/amazon-adsystem\.com/i,
163+
/advertising\.com/i,
164+
165+
// Cookie consent / GDPR
166+
/cookiebot\.com/i,
167+
/cdn\.cookielaw\.org/i,
168+
/consent\.cookiebot\.com/i,
169+
/onetrust\.com/i,
170+
/quantcast\.com/i,
171+
172+
// Push notifications
173+
/onesignal\.com/i,
174+
/pushwoosh\.com/i,
175+
/pusher\.com/i,
176+
177+
// A/B testing
178+
/cdn\.split\.io/i,
179+
/launchdarkly\.com/i,
180+
181+
// Performance monitoring
182+
/speedcurve\.com/i,
183+
/rum\.speedcurve\.com/i,
184+
185+
// Generic patterns
186+
/\/analytics/i,
187+
/\/telemetry/i,
188+
/\/metrics/i,
189+
/\/stats\//i,
190+
];
191+
192+
/**
193+
* Script src patterns to remove from HTML
194+
* These scripts should be removed entirely from the captured HTML
195+
*/
196+
export const STRIP_SCRIPT_PATTERNS = [
197+
// Google
198+
/google-analytics\.com/i,
199+
/googletagmanager\.com/i,
200+
/gtag\/js/i,
201+
/ga\.js/i,
202+
/analytics\.js/i,
203+
204+
// Facebook
205+
/connect\.facebook\.net/i,
206+
/fbevents\.js/i,
207+
208+
// Cloudflare
209+
/cloudflareinsights\.com/i,
210+
/cdn-cgi\/scripts/i,
211+
212+
// Generic analytics
213+
/hotjar\.com/i,
214+
/mixpanel\.com/i,
215+
/segment\.com/i,
216+
/amplitude\.com/i,
217+
/heap-analytics/i,
218+
/intercom\.io/i,
219+
/drift/i,
220+
/hs-scripts\.com/i,
221+
/tawk\.to/i,
222+
/livechatinc\.com/i,
223+
/sentry/i,
224+
/logrocket/i,
225+
/fullstory/i,
226+
/crazyegg/i,
227+
/optimizely/i,
228+
/newrelic/i,
229+
/datadoghq/i,
230+
];
231+
232+
/**
233+
* Inline script content patterns to remove
234+
* These patterns match content inside <script> tags that should be removed
235+
*/
236+
export const STRIP_INLINE_SCRIPT_PATTERNS = [
237+
// Google Analytics initialization
238+
/gtag\s*\(\s*['"]config['"]/i,
239+
/ga\s*\(\s*['"]create['"]/i,
240+
/ga\s*\(\s*['"]send['"]/i,
241+
/_gaq\.push/i,
242+
/GoogleAnalyticsObject/i,
243+
244+
// Facebook Pixel
245+
/fbq\s*\(\s*['"]init['"]/i,
246+
/fbq\s*\(\s*['"]track['"]/i,
247+
248+
// Generic tracking
249+
/window\._analytics/i,
250+
/window\.analytics\.track/i,
251+
/window\.dataLayer/i,
252+
/dataLayer\.push/i,
253+
254+
// Hotjar
255+
/hjid/i,
256+
/hotjar/i,
257+
258+
// Intercom
259+
/window\.intercomSettings/i,
260+
/Intercom\s*\(/i,
261+
262+
// Drift
263+
/drift\.load/i,
264+
265+
// HubSpot
266+
/_hsq\.push/i,
267+
];
268+
269+
/**
270+
* Image/pixel patterns to remove (tracking pixels)
271+
*/
272+
export const STRIP_PIXEL_PATTERNS = [
273+
/facebook\.com\/tr/i,
274+
/pixel\.facebook\.com/i,
275+
/google-analytics\.com\/__utm/i,
276+
/bat\.bing\.com/i,
277+
/analytics\.twitter\.com/i,
278+
/linkedin\.com\/px/i,
279+
/t\.co\/i\/adsct/i,
280+
];
281+
282+
/**
283+
* Check if a URL should be excluded from capture
284+
*/
285+
export function shouldExcludeUrl(url) {
286+
if (!url) return false;
287+
288+
return EXCLUDE_URL_PATTERNS.some(pattern => pattern.test(url));
289+
}
290+
291+
/**
292+
* Check if a script src should be stripped from HTML
293+
*/
294+
export function shouldStripScript(src) {
295+
if (!src) return false;
296+
297+
return STRIP_SCRIPT_PATTERNS.some(pattern => pattern.test(src));
298+
}
299+
300+
/**
301+
* Check if inline script content should be removed
302+
*/
303+
export function shouldStripInlineScript(content) {
304+
if (!content) return false;
305+
306+
return STRIP_INLINE_SCRIPT_PATTERNS.some(pattern => pattern.test(content));
307+
}
308+
309+
/**
310+
* Check if an image is a tracking pixel
311+
*/
312+
export function isTrackingPixel(src) {
313+
if (!src) return false;
314+
315+
return STRIP_PIXEL_PATTERNS.some(pattern => pattern.test(src));
316+
}
317+
318+
/**
319+
* Get a human-readable reason for why a URL was excluded
320+
*/
321+
export function getExcludeReason(url) {
322+
if (!url) return null;
323+
324+
if (/\.map$/i.test(url)) return 'source-map';
325+
if (/\/\.well-known\//i.test(url)) return 'well-known';
326+
if (/cdn-cgi/i.test(url)) return 'cloudflare-service';
327+
if (/google.*analytics|gtag|ga\.js/i.test(url)) return 'google-analytics';
328+
if (/facebook|fbevents/i.test(url)) return 'facebook-tracking';
329+
if (/twitter|t\.co/i.test(url)) return 'twitter-tracking';
330+
if (/hotjar|mixpanel|segment|amplitude|heap/i.test(url)) return 'analytics';
331+
if (/intercom|drift|hubspot|zendesk|crisp|tawk|livechat/i.test(url)) {
332+
return 'chat-widget';
333+
}
334+
if (/sentry|logrocket|fullstory|newrelic|datadog/i.test(url)) {
335+
return 'error-monitoring';
336+
}
337+
if (/beacon|collect\?|pixel\?|track\?/i.test(url)) return 'tracking-beacon';
338+
if (/\/api\/|graphql|webhook/i.test(url)) return 'api-endpoint';
339+
if (/ad\.|adsystem|advertising/i.test(url)) return 'advertising';
340+
if (/cookiebot|cookielaw|onetrust|quantcast/i.test(url))
341+
return 'cookie-consent';
342+
343+
return 'excluded-pattern';
344+
}

0 commit comments

Comments
 (0)