Skip to content
20% OFFCode ZENTIS20 = 20% offSign Up
Back to Blog

AI

AI-Powered Web Monitoring: Detect Changes Before They Impact Your Business

ZentisLabs Team11 min read

Traditional web monitoring tools check if a page is "up" or "down." But modern businesses need to detect meaningful changes: a competitor adjusting prices, a regulatory update, a product going out of stock, or a fraudulent listing appearing. By combining proxies for reliable data collection with LLMs for intelligent analysis, you can build monitoring pipelines that understand context and alert you only when it matters.

Architecture Overview

An AI-powered monitoring pipeline has four stages:

  • Collection: Fetch pages through rotating proxies to avoid blocks
  • Diffing: Compare current content against the last snapshot
  • Analysis: Use an LLM to classify whether the change is meaningful
  • Alerting: Notify via Slack, email, or webhook when action is needed

Step 1: Reliable Collection with Proxies

python
import requests
from bs4 import BeautifulSoup
import hashlib
import json
from datetime import datetime


def fetch_page(url, country="us"):
    """Fetch a page through ZentisLabs proxy with geo-targeting."""
    proxy = f"http://USER:PASS_country-{country}@basic.resi.zentislabs.com:12321"
    proxies = {"http": proxy, "https": proxy}
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/122.0.0.0"}
    
    r = requests.get(url, proxies=proxies, headers=headers, timeout=15)
    r.raise_for_status()
    
    soup = BeautifulSoup(r.text, "html.parser")
    # Remove dynamic elements (ads, timestamps)
    for tag in soup.select("script, style, iframe, .ad-banner, .timestamp"):
        tag.decompose()
    
    return {
        "url": url,
        "text": soup.get_text(separator=" ", strip=True),
        "html": str(soup),
        "hash": hashlib.sha256(soup.get_text().encode()).hexdigest(),
        "fetched_at": datetime.utcnow().isoformat(),
    }

Step 2: Smart Diffing

python
import difflib


def compute_diff(old_text, new_text):
    """Compute a human-readable diff between two page snapshots."""
    old_lines = old_text.splitlines()
    new_lines = new_text.splitlines()
    
    diff = difflib.unified_diff(old_lines, new_lines, lineterm="")
    changes = "\n".join(diff)
    
    # Calculate change percentage
    matcher = difflib.SequenceMatcher(None, old_text, new_text)
    similarity = matcher.ratio()
    change_pct = round((1 - similarity) * 100, 1)
    
    return {"diff": changes, "change_pct": change_pct, "has_changes": change_pct > 0.5}

Step 3: LLM Analysis

python
import openai


def analyze_change(url, diff_text, change_pct):
    """Use an LLM to determine if the change is meaningful."""
    client = openai.OpenAI()
    
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "You analyze website changes. Classify each change as: CRITICAL (pricing, legal, product changes), NOTABLE (new content, layout changes), or NOISE (timestamp, ad rotation, minor formatting). Respond with JSON: {category, summary, action_needed}."},
            {"role": "user", "content": f"URL: {url}\nChange: {change_pct}%\n\nDiff:\n{diff_text[:3000]}"}
        ],
        response_format={"type": "json_object"},
    )
    return json.loads(response.choices[0].message.content)

Step 4: Smart Alerting

python
import requests as req


def send_alert(analysis, url):
    """Send alert to Slack if change is meaningful."""
    if analysis["category"] == "NOISE":
        return  # Skip noise
    
    color = "#ff0000" if analysis["category"] == "CRITICAL" else "#ffaa00"
    
    payload = {
        "attachments": [{
            "color": color,
            "title": f"{analysis['category']}: {url}",
            "text": analysis["summary"],
            "fields": [
                {"title": "Action Needed", "value": analysis.get("action_needed", "Review"), "short": True},
            ],
        }]
    }
    req.post("https://hooks.slack.com/services/YOUR/WEBHOOK/URL", json=payload)

Putting It Together

python
# monitor.py - Run on a schedule (cron, Airflow, n8n)
import json
from pathlib import Path


def monitor_url(url, country="us"):
    snapshot_file = Path(f"snapshots/{hashlib.md5(url.encode()).hexdigest()}.json")
    
    # Fetch current page
    current = fetch_page(url, country)
    
    # Load previous snapshot
    if snapshot_file.exists():
        previous = json.loads(snapshot_file.read_text())
        diff = compute_diff(previous["text"], current["text"])
        
        if diff["has_changes"]:
            analysis = analyze_change(url, diff["diff"], diff["change_pct"])
            send_alert(analysis, url)
            print(f"[{analysis['category']}] {url}: {analysis['summary']}")
    
    # Save current snapshot
    snapshot_file.parent.mkdir(exist_ok=True)
    snapshot_file.write_text(json.dumps(current))


# Monitor competitor pricing pages
urls = [
    "https://competitor1.com/pricing",
    "https://competitor2.com/plans",
    "https://competitor3.com/api/pricing",
]
for url in urls:
    monitor_url(url)

ZentisLabs residential proxies keep your monitoring running on fresh IPs from a 100M+ pool. Rotate IPs automatically and geo-target to see the same content your customers see in any market.

Ready to get started?

Try ZentisLabs free - 100MB residential bandwidth included.

Get 100MB Free

Related Articles