Google Gemini Web Scraping API: Complete Integration Guide

# Install required packages
pip install google-generativeai supacrawler requests python-dotenv

def analyze_article_with_gemini(url):
    """Scrape an article and analyze it with Gemini"""
    
    # Step 1: Scrape the content
    result = supacrawler.scrape(url, format="markdown")
    
    if not result.markdown:
        return {"error": "Failed to scrape content"}
    
    # Step 2: Analyze with Gemini
    prompt = f"""
    Analyze this article and extract key information:
    
    Article Content:
    {result.markdown[:4000]}
    
    Please provide:
    1. Main topic/subject
    2. Key points (max 5)
    3. Sentiment (positive/negative/neutral)
    4. Article category
    5. Target audience
    
    Format as JSON.
    """
    
    response = gemini_model.generate_content(prompt)
    
    try:
        analysis = json.loads(response.text)
        return {
            "url": url,
            "analysis": analysis,
            "success": True
        }
    except json.JSONDecodeError:
        return {
            "url": url,
            "analysis": response.text,
            "success": True
        }

# Example usage
url = "https://techcrunch.com/latest-article"
result = analyze_article_with_gemini(url)
print(json.dumps(result["analysis"], indent=2))

def extract_financial_insights(company_url):
    """Extract and analyze financial information with context"""
    
    # Scrape company financial page
    result = supacrawler.scrape(company_url, format="markdown", render_js=True)
    
    if not result.markdown:
        return {"error": "Failed to scrape financial data"}
    
    prompt = f"""
    Analyze this financial information and extract key insights:
    
    {result.markdown[:8000]}
    
    Provide analysis in this JSON format:
    {{
        "company_metrics": {{
            "revenue": "latest revenue figure",
            "profit_margin": "profit margin percentage",
            "growth_rate": "year-over-year growth"
        }},
        "financial_health": {{
            "score": "1-10 scale",
            "key_strengths": ["strength1", "strength2"],
            "risk_factors": ["risk1", "risk2"]
        }},
        "key_numbers": [
            {{"metric": "name", "value": "number", "context": "explanation"}}
        ]
    }}
    
    Focus on extracting actual numbers and providing meaningful analysis.
    """
    
    response = gemini_model.generate_content(prompt)
    
    try:
        insights = json.loads(response.text)
        return {
            "success": True,
            "url": company_url,
            "financial_insights": insights
        }
    except json.JSONDecodeError:
        return {
            "success": False,
            "error": "Failed to parse financial analysis",
            "raw_response": response.text
        }

# Example usage
company_url = "https://investor-relations.example-company.com"
financial_data = extract_financial_insights(company_url)
print(json.dumps(financial_data, indent=2))

def monitor_competitor_prices(competitor_urls):
    """Monitor competitor prices with intelligent analysis"""
    
    results = []
    
    for url in competitor_urls:
        # Scrape competitor page
        result = supacrawler.scrape(url, format="markdown", render_js=True)
        
        if not result.markdown:
            results.append({
                "url": url,
                "error": "Failed to scrape",
                "success": False
            })
            continue
        
        # Extract pricing information with Gemini
        prompt = f"""
        Extract pricing information from this e-commerce page:
        
        {result.markdown[:4000]}
        
        Find and extract:
        {{
            "pricing_found": true/false,
            "pricing_model": "subscription/one-time/freemium/etc",
            "price_points": [
                {{"tier": "name", "price": "amount", "currency": "code", "features": ["feature1"]}}
            ],
            "company_name": "company name if identified",
            "pricing_strategy": "premium/budget/competitive/etc",
            "special_offers": ["offer1", "offer2"]
        }}
        """
        
        response = gemini_model.generate_content(prompt)
        
        try:
            pricing_info = json.loads(response.text)
            results.append({
                "url": url,
                "pricing": pricing_info,
                "success": pricing_info.get('pricing_found', False),
                "scraped_at": "2025-01-03"
            })
        except json.JSONDecodeError:
            results.append({
                "url": url,
                "raw_analysis": response.text,
                "success": False
            })
    
    return {
        "competitors_analyzed": len(results),
        "pricing_data": results,
        "successful_extractions": len([r for r in results if r["success"]])
    }

# Example usage
competitor_urls = [
    "https://competitor1.com/pricing",
    "https://competitor2.com/plans",
    "https://competitor3.com/pricing"
]

pricing_analysis = monitor_competitor_prices(competitor_urls)
print(f"Analyzed {pricing_analysis['competitors_analyzed']} competitors")
print(f"Successful extractions: {pricing_analysis['successful_extractions']}")

def optimize_content_for_gemini(content, max_tokens=4000):
    """Optimize content before sending to Gemini to reduce token usage"""
    
    # Remove common noise patterns
    import re
    
    # Remove excessive whitespace
    content = re.sub(r'\s+', ' ', content)
    
    # Remove common noise patterns
    noise_patterns = [
        r'Cookie Policy.*?(?=\n|$)',
        r'Privacy Policy.*?(?=\n|$)',
        r'Accept.*?cookies.*?(?=\n|$)',
        r'\d{4}.*?All rights reserved',
        r'Subscribe to.*?newsletter',
    ]
    
    for pattern in noise_patterns:
        content = re.sub(pattern, '', content, flags=re.IGNORECASE)
    
    # Truncate to max tokens (rough estimate: 1 token ≈ 4 characters)
    max_chars = max_tokens * 4
    if len(content) > max_chars:
        content = content[:max_chars] + "..."
    
    return content.strip()

def batch_analyze_with_caching(urls, cache_duration=3600):
    """Batch process URLs with intelligent caching"""
    
    import hashlib
    import time
    
    cache = {}
    results = []
    
    for url in urls:
        # Create cache key based on URL and timestamp
        cache_key = hashlib.md5(f"{url}:{int(time.time() // cache_duration)}".encode()).hexdigest()
        
        if cache_key in cache:
            results.append(cache[cache_key])
            continue
        
        # Process new URL
        result = supacrawler.scrape(url, format="markdown")
        if result.markdown:
            content = optimize_content_for_gemini(result.markdown)
            
            # Use focused prompt to reduce response tokens
            prompt = f"""
            Briefly analyze this content (max 100 words):
            {content}
            
            Return only: {{"topic": "main topic", "sentiment": "pos/neg/neu", "type": "content type"}}
            """
            
            response = gemini_model.generate_content(prompt)
            processed_result = {
                "url": url,
                "analysis": response.text,
                "cached": False
            }
        else:
            processed_result = {
                "url": url,
                "error": "Failed to scrape",
                "cached": False
            }
        
        cache[cache_key] = processed_result
        results.append(processed_result)
    
    return results

# Example usage
urls = [
    "https://example.com/page1",
    "https://example.com/page2",
    "https://example.com/page3"
]

results = batch_analyze_with_caching(urls)
print(f"Processed {len(results)} URLs with caching")