99import logging
1010import re
1111import socket
12+ import threading
1213from dataclasses import dataclass
1314from datetime import datetime , timezone
1415from email .utils import parsedate_to_datetime
1718from xml .etree import ElementTree as ET
1819
1920import requests
21+ from sqlalchemy .exc import IntegrityError
2022
2123from src .config import get_config
2224from src .repositories .intelligence_repo import IntelligenceRepository
3133_MAX_FEED_BYTES = 2 * 1024 * 1024
3234_MAX_FEED_REDIRECTS = 5
3335_REDIRECT_STATUS_CODES = {301 , 302 , 303 , 307 , 308 }
36+ _DISABLE_REQUEST_PROXIES = {"http" : None , "https" : None }
37+ _DNS_GUARD_LOCK = threading .Lock ()
38+ _BUILTIN_SOURCE_TEMPLATES = [
39+ {
40+ "template_id" : "sec-company-news" ,
41+ "name" : "SEC Latest Filings" ,
42+ "source_type" : "rss" ,
43+ "url" : "https://www.sec.gov/news/pressreleases.rss" ,
44+ "scope_type" : "market" ,
45+ "market" : "us" ,
46+ "description" : "SEC official press release RSS feed for US market evidence." ,
47+ },
48+ {
49+ "template_id" : "hkex-news" ,
50+ "name" : "HKEX Market News" ,
51+ "source_type" : "rss" ,
52+ "url" : "https://www.hkex.com.hk/Services/RSS-Feeds/News-Releases?sc_lang=en" ,
53+ "scope_type" : "market" ,
54+ "market" : "hk" ,
55+ "description" : "HKEX public news entry for Hong Kong market evidence. Test before enabling." ,
56+ },
57+ {
58+ "template_id" : "global-marketwatch" ,
59+ "name" : "MarketWatch Top Stories" ,
60+ "source_type" : "rss" ,
61+ "url" : "https://feeds.content.dowjones.io/public/rss/mw_topstories" ,
62+ "scope_type" : "market" ,
63+ "market" : "global" ,
64+ "description" : "Public market news RSS for global market context. Test before enabling." ,
65+ },
66+ ]
3467
3568
3669class IntelligenceServiceError (ValueError ):
@@ -57,7 +90,10 @@ def __init__(self, repository: Optional[IntelligenceRepository] = None):
5790 def create_source (self , payload : Dict [str , Any ]) -> Dict [str , Any ]:
5891 fields = self ._normalize_source_fields (payload )
5992 self ._validate_url (fields ["url" ])
60- return self ._source_to_dict (self .repo .create_source (fields ))
93+ try :
94+ return self ._source_to_dict (self .repo .create_source (fields ))
95+ except IntegrityError as exc :
96+ raise IntelligenceServiceError (f"intelligence source name already exists: { fields ['name' ]} ) from exc
6197
6298 def list_sources (self , ** filters : Any ) -> Dict [str , Any ]:
6399 rows , total = self .repo .list_sources (** filters )
@@ -68,6 +104,29 @@ def list_sources(self, **filters: Any) -> Dict[str, Any]:
68104 "page_size" : max (1 , min (int (filters .get ("page_size" ) or 50 ), 100 )),
69105 }
70106
107+ def list_source_templates (self , ** filters : Any ) -> Dict [str , Any ]:
108+ market = str (filters .get ("market" ) or "" ).strip ().lower ()
109+ source_type = str (filters .get ("source_type" ) or "" ).strip ().lower ()
110+ templates = []
111+ for template in _BUILTIN_SOURCE_TEMPLATES :
112+ if market and template ["market" ] != market :
113+ continue
114+ if source_type and template ["source_type" ] != source_type :
115+ continue
116+ templates .append (dict (template ))
117+ return {"items" : templates , "total" : len (templates )}
118+
119+ def create_source_from_template (self , template_id : str , overrides : Optional [Dict [str , Any ]] = None ) -> Dict [str , Any ]:
120+ selected = next (
121+ (dict (template ) for template in _BUILTIN_SOURCE_TEMPLATES if template ["template_id" ] == template_id ),
122+ None ,
123+ )
124+ if selected is None :
125+ raise IntelligenceServiceError (f"Intelligence source template not found: { template_id } )
126+ payload = {key : value for key , value in selected .items () if key != "template_id" }
127+ payload .update ({key : value for key , value in (overrides or {}).items () if value is not None })
128+ return self .create_source (payload )
129+
71130 def list_items (self , ** filters : Any ) -> Dict [str , Any ]:
72131 rows , total = self .repo .list_items (** filters )
73132 return {
@@ -193,7 +252,7 @@ def _validate_url(self, raw_url: str, *, allow_no_url: bool = False) -> None:
193252 except ValueError :
194253 ip = None
195254 if ip is not None :
196- if ip . is_private or ip . is_loopback or ip . is_link_local or ip . is_reserved or ip . is_multicast :
255+ if self . _is_blocked_ip ( ip ) :
197256 raise IntelligenceServiceError ("source url must not target private or local network addresses" )
198257 return
199258 try :
@@ -207,12 +266,23 @@ def _validate_url(self, raw_url: str, *, allow_no_url: bool = False) -> None:
207266 ip = ipaddress .ip_address (info [4 ][0 ])
208267 except (IndexError , ValueError ):
209268 continue
210- if ip . is_private or ip . is_loopback or ip . is_link_local or ip . is_reserved or ip . is_multicast :
269+ if self . _is_blocked_ip ( ip ) :
211270 raise IntelligenceServiceError ("source url must not target private or local network addresses" )
212271 has_public_address = True
213272 if not has_public_address :
214273 raise IntelligenceServiceError (f"source url host DNS resolution failed: { hostname } )
215274
275+ @staticmethod
276+ def _is_blocked_ip (ip : ipaddress ._BaseAddress ) -> bool :
277+ return (
278+ not ip .is_global
279+ or ip .is_private
280+ or ip .is_loopback
281+ or ip .is_link_local
282+ or ip .is_reserved
283+ or ip .is_multicast
284+ )
285+
216286 def _fetch_feed_entries (self , fields : Dict [str , Any ], * , limit : int ) -> List [FeedEntry ]:
217287 timeout = max (1 , min (float (self .config .news_intel_fetch_timeout_sec ), 30.0 ))
218288 headers = {"User-Agent" : "daily-stock-analysis-intel/1.0" }
@@ -221,13 +291,12 @@ def _fetch_feed_entries(self, fields: Dict[str, Any], *, limit: int) -> List[Fee
221291 response = None
222292 try :
223293 for _ in range (_MAX_FEED_REDIRECTS + 1 ):
224- response = requests . get (
294+ response = self . _get_with_validated_dns (
225295 request_url ,
226296 timeout = timeout ,
227297 headers = headers ,
228298 allow_redirects = False ,
229299 stream = True ,
230- trust_env = False ,
231300 )
232301 status_code = int (getattr (response , "status_code" , 200 ))
233302 if status_code in _REDIRECT_STATUS_CODES :
@@ -269,6 +338,46 @@ def _fetch_feed_entries(self, fields: Dict[str, Any], *, limit: int) -> List[Fee
269338 if response is not None :
270339 response .close ()
271340
341+ def _get_with_validated_dns (self , raw_url : str , ** kwargs : Any ) -> requests .Response :
342+ parsed = urlparse (raw_url )
343+ target_hostname = self ._normalize_hostname (parsed .hostname )
344+ original_getaddrinfo = socket .getaddrinfo
345+
346+ def guarded_getaddrinfo (host : Any , port : Any , * args : Any , ** inner_kwargs : Any ) -> Any :
347+ addrinfos = original_getaddrinfo (host , port , * args , ** inner_kwargs )
348+ if self ._normalize_hostname (host ) == target_hostname :
349+ self ._validate_addrinfos (addrinfos )
350+ return addrinfos
351+
352+ with _DNS_GUARD_LOCK :
353+ socket .getaddrinfo = guarded_getaddrinfo
354+ try :
355+ request_kwargs = dict (kwargs )
356+ request_kwargs .setdefault ("proxies" , _DISABLE_REQUEST_PROXIES )
357+ return requests .get (raw_url , ** request_kwargs )
358+ finally :
359+ socket .getaddrinfo = original_getaddrinfo
360+
361+ @staticmethod
362+ def _normalize_hostname (hostname : Any ) -> str :
363+ if isinstance (hostname , bytes ):
364+ hostname = hostname .decode ("ascii" , errors = "ignore" )
365+ normalized = str (hostname or "" ).strip ().lower ().rstrip ("." )
366+ try :
367+ return normalized .encode ("idna" ).decode ("ascii" )
368+ except UnicodeError :
369+ return normalized
370+
371+ @staticmethod
372+ def _validate_addrinfos (addr_infos : Any ) -> None :
373+ for info in addr_infos or []:
374+ try :
375+ ip = ipaddress .ip_address (info [4 ][0 ])
376+ except (IndexError , TypeError , ValueError ):
377+ continue
378+ if IntelligenceService ._is_blocked_ip (ip ):
379+ raise IntelligenceServiceError ("source url must not target private or local network addresses" )
380+
272381 def _parse_feed (self , content : bytes , * , source_name : str , limit : int ) -> List [FeedEntry ]:
273382 try :
274383 root = ET .fromstring (content )
@@ -313,7 +422,10 @@ def _build_entry(self, title: str, summary: str, url: str, source_name: str, pub
313422 if not title and not url :
314423 return None
315424 if url :
316- self ._validate_url (url , allow_no_url = True )
425+ try :
426+ self ._validate_url (url , allow_no_url = True )
427+ except IntelligenceServiceError :
428+ return None
317429 url_key = url
318430 else :
319431 digest = hashlib .sha256 (f"{ source_name } { title } { published_at } .encode ("utf-8" )).hexdigest ()[:24 ]
0 commit comments