Strip tracking parameters from urls and html content

Author: tsubery

app/models/media_item.rb

@@ -12,6 +12,7 @@ class MediaItem < ApplicationRecord
   before_save :widen_substack_email
   before_save :widen_bloomberg_email
   before_save :embed_images_and_resolve_links
+  before_save :strip_tracking_params
   # after_save :cache_article
 
   TEMPORARY_URL = "https://temporary.local".freeze
@@ -29,6 +30,22 @@ class MediaItem < ApplicationRecord
   MISSING_DURATION_THRESHOLD = 5
   EMAIL_WIDE_LAYOUT_PX = 950
 
+  TRACKING_PARAM_PATTERNS = [
+    /\Autm_/i,         # Google Analytics — utm_source, utm_medium, utm_campaign, ...
+    /\Asi\z/i,         # YouTube share id
+    /\Afbclid\z/i,     # Facebook click id
+    /\Agclid\z/i,      # Google ads click id
+    /\Agbraid\z/i,     # Google ads (iOS app)
+    /\Awbraid\z/i,     # Google ads (web)
+    /\Amc_(cid|eid)\z/i, # Mailchimp
+    /\A_hsenc\z/i,     # HubSpot
+    /\A_hsmi\z/i,      # HubSpot
+    /\AhsCtaTracking\z/i, # HubSpot
+    /\Aigshid\z/i,     # Instagram share id
+    /\Ayclid\z/i,      # Yandex
+    /\Amsclkid\z/i,    # Microsoft ads
+  ].freeze
+
   scope :articles, -> { where(mime_type: HTML_MIME_TYPE) }
   scope :unarchived, -> { where(archived: false) }
   scope :reading_list, -> { articles.unarchived.joins(:feed, :libraries).order("feeds.priority ASC, media_items.created_at DESC") }
@@ -152,6 +169,35 @@ def widen_bloomberg_email
     self.description = doc.to_html
   end
 
+  def self.strip_tracking_params(url)
+    return url if url.blank?
+    uri = URI.parse(url)
+    return url if uri.query.blank?
+    params = URI.decode_www_form(uri.query).reject do |key, _|
+      TRACKING_PARAM_PATTERNS.any? { |p| key.match?(p) }
+    end
+    uri.query = params.empty? ? nil : URI.encode_www_form(params)
+    uri.to_s
+  rescue URI::InvalidURIError
+    url
+  end
+
+  def strip_tracking_params
+    self.url = self.class.strip_tracking_params(url)
+    return unless html?
+
+    doc = Nokogiri::HTML(description)
+    changed = false
+    doc.css('a[href]').each do |link|
+      cleaned = self.class.strip_tracking_params(link['href'])
+      if cleaned != link['href']
+        link['href'] = cleaned
+        changed = true
+      end
+    end
+    self.description = doc.to_html if changed
+  end
+
   def embed_images_and_resolve_links
     return unless html?
 

test/models/media_item_test.rb

@@ -300,4 +300,78 @@ def create_stubbed_video(overrides)
     assert_not article.video?
     assert_not article.video_cached?
   end
+
+  test "strip_tracking_params removes utm_* keys" do
+    url = "https://example.com/page?utm_source=x&utm_medium=y&utm_campaign=z&kept=1"
+    assert_equal "https://example.com/page?kept=1", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params removes youtube si parameter" do
+    url = "https://www.youtube.com/watch?v=abc123&si=AbCdEf"
+    assert_equal "https://www.youtube.com/watch?v=abc123", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params removes other known trackers" do
+    %w[fbclid gclid gbraid wbraid mc_cid mc_eid _hsenc _hsmi hsCtaTracking igshid yclid msclkid].each do |key|
+      url = "https://example.com/page?#{key}=value&kept=1"
+      assert_equal "https://example.com/page?kept=1", MediaItem.strip_tracking_params(url),
+                   "expected #{key} to be stripped"
+    end
+  end
+
+  test "strip_tracking_params preserves non-tracking parameters" do
+    url = "https://example.com/page?id=42&q=hello"
+    assert_equal "https://example.com/page?id=42&q=hello", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params returns url unchanged when query is empty" do
+    url = "https://example.com/page"
+    assert_equal "https://example.com/page", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params drops the query string when all params are trackers" do
+    url = "https://example.com/page?utm_source=a&fbclid=b"
+    assert_equal "https://example.com/page", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params preserves fragment after stripping all params" do
+    url = "https://example.com/page?utm_source=a#section"
+    assert_equal "https://example.com/page#section", MediaItem.strip_tracking_params(url)
+  end
+
+  test "strip_tracking_params returns blank input unchanged" do
+    assert_nil MediaItem.strip_tracking_params(nil)
+    assert_equal "", MediaItem.strip_tracking_params("")
+  end
+
+  test "strip_tracking_params returns invalid url unchanged" do
+    bad = "http://exa mple.com/?utm_source=x"
+    assert_equal bad, MediaItem.strip_tracking_params(bad)
+  end
+
+  test "before_save strips tracking params from url" do
+    feed = feeds(:one)
+    mi = feed.media_items.create!(
+      url: "https://example.com/post?utm_source=newsletter&id=42",
+      guid: "https://example.com/post-#{SecureRandom.hex}",
+      title: "T",
+      description: "plain text",
+      mime_type: MediaItem::HTML_MIME_TYPE
+    )
+    assert_equal "https://example.com/post?id=42", mi.url
+  end
+
+  test "before_save strips tracking params from anchor hrefs in description" do
+    feed = feeds(:one)
+    description = '<html><body><a href="https://www.youtube.com/watch?v=abc&si=tracker">v</a></body></html>'
+    mi = feed.media_items.create!(
+      url: "https://example.com/post",
+      guid: "https://example.com/post-anchor-#{SecureRandom.hex}",
+      title: "T",
+      description: description,
+      mime_type: MediaItem::HTML_MIME_TYPE
+    )
+    assert_includes mi.description, 'href="https://www.youtube.com/watch?v=abc"'
+    assert_not_includes mi.description, "si=tracker"
+  end
 end