requirements.txt

# Core Dependencies
scrapy==2.11.0
selenium==4.17.2
pandas==2.2.0
openpyxl==3.1.2
pytesseract==0.3.10
python-dotenv==1.0.1
requests==2.31.0
tldextract==5.1.1
validators==0.22.0
pydantic>=2.6.1
duckduckgo-search>=4.1.1

# Scraping Dependencies
scrapy-splash==0.9.0
scrapy-user-agents==0.1.1
fake-useragent==1.4.0
beautifulsoup4==4.12.3
lxml==5.1.0
w3lib>=2.1.2
parsel>=1.8.1

# Database
psycopg2-binary==2.9.9
SQLAlchemy==2.0.25
redis>=3.0.0,<4.0.0
python-memcached>=1.59

# Cache & Compression
msgpack==1.0.7
# python-snappy==0.6.1
lz4==4.3.2
zstandard==0.22.0
redis-py-cluster>=2.1.3,<3.0.0
uhashring==2.3
zlib-state>=0.1.5
python-magic==0.4.27

# Image Processing
Pillow==10.2.0
opencv-python==4.9.0.80

# UI Dependencies
streamlit==1.30.0
plotly==5.18.0
altair==5.2.0
streamlit-option-menu==0.3.6
streamlit-aggrid==0.3.4

# Utilities
python-dateutil>=2.8.2
pytz==2024.1
pyyaml==6.0.1
cryptography>=39.0.0
retry==0.9.2
typing-extensions>=4.5.0

# Monitoring & Logging
prometheus-client>=0.16.0
sentry-sdk==1.40.0
diskcache==5.6.3
psutil==5.9.8

# Testing
pytest>=7.3.1
pytest-cov==4.1.0
pytest-mock>=3.0
responses>=0.23

# Code Quality
black==24.1.1
flake8==7.0.0
mypy==1.8.0
isort>=5.0
pre-commit>=3.0

# Documentation
sphinx>=7.1.2
sphinx-rtd-theme>=1.3.0
sphinx-autodoc-typehints>=1.25.2

# Additional dependencies
webdriver-manager==4.0.1
pyarrow>=14.0.1