Playwright vs Selenium vs Puppeteer for Web Scraping in 2025: The Complete Guide

# Python Example
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options

# Configure Chrome options for scraping
options = Options()
options.add_argument('--headless')  # Run in background
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

# Initialize driver
driver = webdriver.Chrome(options=options)

try:
    # Navigate to target page
    driver.get('https://example.com/products')
    
    # Wait for dynamic content to load
    wait = WebDriverWait(driver, 10)
    products = wait.until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, 'product-card'))
    )
    
    # Extract data
    scraped_data = []
    for product in products:
        title = product.find_element(By.CLASS_NAME, 'product-title').text
        price = product.find_element(By.CLASS_NAME, 'product-price').text
        
        scraped_data.append({
            'title': title,
            'price': price
        })
    
    print(f"Scraped {len(scraped_data)} products")
    
finally:
    driver.quit()

// JavaScript Example (using selenium-webdriver npm package)
const { Builder, By, until } = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');

async function scrapeProducts() {
    // Configure Chrome options
    const options = new chrome.Options();
    options.addArguments('--headless');
    options.addArguments('--disable-blink-features=AutomationControlled');
    options.excludeSwitches('enable-automation');
    options.setUserPreferences({ 'useAutomationExtension': false });

    // Build driver
    const driver = await new Builder()
        .forBrowser('chrome')
        .setChromeOptions(options)
        .build();

    try {
        // Navigate to page
        await driver.get('https://example.com/products');
        
        // Wait for products to load
        await driver.wait(until.elementsLocated(By.className('product-card')), 10000);
        const products = await driver.findElements(By.className('product-card'));
        
        // Extract data
        const scrapedData = [];
        for (const product of products) {
            const title = await product.findElement(By.className('product-title')).getText();
            const price = await product.findElement(By.className('product-price')).getText();
            
            scrapedData.push({ title, price });
        }
        
        console.log(`Scraped ${scrapedData.length} products`);
        return scrapedData;
        
    } finally {
        await driver.quit();
    }
}

scrapeProducts();

// Java Example
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.WebDriverWait;
import org.openqa.selenium.support.ui.ExpectedConditions;
import java.time.Duration;
import java.util.*;

public class SeleniumScraper {
    public static void main(String[] args) {
        // Configure Chrome options
        ChromeOptions options = new ChromeOptions();
        options.addArguments("--headless");
        options.addArguments("--disable-blink-features=AutomationControlled");
        options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
        options.setExperimentalOption("useAutomationExtension", false);
        
        // Initialize driver
        WebDriver driver = new ChromeDriver(options);
        
        try {
            // Navigate to page
            driver.get("https://example.com/products");
            
            // Wait for products to load
            WebDriverWait wait = new WebDriverWait(driver, Duration.ofSeconds(10));
            List products = wait.until(
                ExpectedConditions.presenceOfAllElementsLocatedBy(By.className("product-card"))
            );
            
            // Extract data
            List> scrapedData = new ArrayList<>();
            for (WebElement product : products) {
                String title = product.findElement(By.className("product-title")).getText();
                String price = product.findElement(By.className("product-price")).getText();
                
                Map item = new HashMap<>();
                item.put("title", title);
                item.put("price", price);
                scrapedData.add(item);
            }
            
            System.out.println("Scraped " + scrapedData.size() + " products");
            
        } finally {
            driver.quit();
        }
    }
}

// JavaScript Example
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

// Add stealth plugin to avoid detection
puppeteer.use(StealthPlugin());

async function scrapeProducts() {
    const browser = await puppeteer.launch({
        headless: 'new',
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-web-security',
            '--disable-features=IsolateOrigins,site-per-process'
        ]
    });

    try {
        const page = await browser.newPage();
        
        // Set realistic viewport
        await page.setViewport({ width: 1920, height: 1080 });
        
        // Navigate with realistic network conditions
        await page.goto('https://example.com/products', {
            waitUntil: 'networkidle2',
            timeout: 30000
        });
        
        // Wait for products to load
        await page.waitForSelector('.product-card', { timeout: 10000 });
        
        // Extract data using page.evaluate
        const products = await page.evaluate(() => {
            return Array.from(document.querySelectorAll('.product-card')).map(card => ({
                title: card.querySelector('.product-title')?.textContent?.trim(),
                price: card.querySelector('.product-price')?.textContent?.trim(),
                image: card.querySelector('img')?.src,
                link: card.querySelector('a')?.href
            }));
        });
        
        console.log(`Scraped ${products.length} products`);
        
        // Take screenshot for debugging
        await page.screenshot({ path: 'products.png', fullPage: true });
        
        return products;
        
    } finally {
        await browser.close();
    }
}

// Handle infinite scrolling
async function scrapeInfiniteScroll() {
    const browser = await puppeteer.launch({ headless: 'new' });
    const page = await browser.newPage();
    
    await page.goto('https://example.com/infinite-scroll');
    
    // Auto-scroll function
    await page.evaluate(async () => {
        await new Promise((resolve) => {
            let totalHeight = 0;
            const distance = 100;
            const timer = setInterval(() => {
                const scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;
                
                if(totalHeight >= scrollHeight){
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        });
    });
    
    // Now scrape all loaded content
    const allItems = await page.$$eval('.item', items => 
        items.map(item => item.textContent)
    );
    
    await browser.close();
    return allItems;
}

// TypeScript Example
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';

// Add stealth plugin
puppeteer.use(StealthPlugin());

interface Product {
    title: string | null;
    price: string | null;
    image: string | undefined;
    link: string | undefined;
}

async function scrapeProducts(): Promise {
    const browser = await puppeteer.launch({
        headless: 'new',
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-web-security'
        ]
    });

    try {
        const page = await browser.newPage();
        
        // Set viewport and user agent
        await page.setViewport({ width: 1920, height: 1080 });
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
        
        // Navigate with type safety
        await page.goto('https://example.com/products', {
            waitUntil: 'networkidle2',
            timeout: 30000
        });
        
        // Wait for content
        await page.waitForSelector('.product-card', { timeout: 10000 });
        
        // Extract with type safety
        const products = await page.evaluate((): Product[] => {
            const cards = document.querySelectorAll('.product-card');
            return Array.from(cards).map((card): Product => ({
                title: card.querySelector('.product-title')?.textContent?.trim() ?? null,
                price: card.querySelector('.product-price')?.textContent?.trim() ?? null,
                image: card.querySelector('img')?.src,
                link: card.querySelector('a')?.href
            }));
        });
        
        console.log(`Scraped ${products.length} products`);
        return products;
        
    } finally {
        await browser.close();
    }
}

// Execute with error handling
scrapeProducts()
    .then(products => console.log(products))
    .catch(error => console.error('Scraping failed:', error));

# Python Example (using pyppeteer)
import asyncio
from pyppeteer import launch
from pyppeteer_stealth import stealth

async def scrape_products():
    # Launch browser with stealth settings
    browser = await launch({
        'headless': True,
        'args': [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-web-security',
            '--disable-features=IsolateOrigins,site-per-process'
        ]
    })
    
    try:
        page = await browser.newPage()
        
        # Apply stealth to avoid detection
        await stealth(page)
        
        # Set viewport
        await page.setViewport({'width': 1920, 'height': 1080})
        
        # Navigate to page
        await page.goto('https://example.com/products', {
            'waitUntil': 'networkidle2',
            'timeout': 30000
        })
        
        # Wait for products to load
        await page.waitForSelector('.product-card', {'timeout': 10000})
        
        # Extract data
        products = await page.evaluate('''() => {
            return Array.from(document.querySelectorAll('.product-card')).map(card => ({
                title: card.querySelector('.product-title')?.textContent?.trim(),
                price: card.querySelector('.product-price')?.textContent?.trim(),
                image: card.querySelector('img')?.src,
                link: card.querySelector('a')?.href
            }));
        }''')
        
        print(f'Scraped {len(products)} products')
        
        # Take screenshot
        await page.screenshot({'path': 'products.png', 'fullPage': True})
        
        return products
        
    finally:
        await browser.close()

# Run the scraper
asyncio.run(scrape_products())

// TypeScript Example
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';

// Add stealth plugin
puppeteer.use(StealthPlugin());

interface Product {
    title: string | null;
    price: string | null;
    image: string | undefined;
    link: string | undefined;
}

async function scrapeProducts(): Promise {
    const browser = await puppeteer.launch({
        headless: 'new',
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-web-security'
        ]
    });

    try {
        const page = await browser.newPage();
        
        // Set viewport and user agent
        await page.setViewport({ width: 1920, height: 1080 });
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
        
        // Navigate with type safety
        await page.goto('https://example.com/products', {
            waitUntil: 'networkidle2',
            timeout: 30000
        });
        
        // Wait for content
        await page.waitForSelector('.product-card', { timeout: 10000 });
        
        // Extract with type safety
        const products = await page.evaluate((): Product[] => {
            const cards = document.querySelectorAll('.product-card');
            return Array.from(cards).map((card): Product => ({
                title: card.querySelector('.product-title')?.textContent?.trim() ?? null,
                price: card.querySelector('.product-price')?.textContent?.trim() ?? null,
                image: card.querySelector('img')?.src,
                link: card.querySelector('a')?.href
            }));
        });
        
        console.log(`Scraped ${products.length} products`);
        return products;
        
    } finally {
        await browser.close();
    }
}

// Execute with error handling
scrapeProducts()
    .then(products => console.log(products))
    .catch(error => console.error('Scraping failed:', error));

// JavaScript Example
const { chromium, devices } = require('playwright');

async function advancedScraping() {
    // Launch with anti-detection measures
    const browser = await chromium.launch({
        headless: false,
        args: [
            '--disable-blink-features=AutomationControlled',
        ]
    });

    // Create context with realistic settings
    const context = await browser.newContext({
        viewport: { width: 1920, height: 1080 },
        userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        locale: 'en-US',
        timezoneId: 'America/New_York',
        permissions: ['geolocation'],
        geolocation: { latitude: 40.7128, longitude: -74.0060 }
    });

    // Block unnecessary resources for faster loading
    await context.route('**/*.{png,jpg,jpeg,gif,webp,svg,ico}', route => route.abort());
    await context.route('**/*.{css,font}', route => route.abort());
    
    const page = await context.newPage();
    
    // Intercept and modify requests
    await page.route('**/*', (route, request) => {
        const headers = {
            ...request.headers(),
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Encoding': 'gzip, deflate, br',
            'Cache-Control': 'no-cache',
            'Pragma': 'no-cache'
        };
        route.continue({ headers });
    });

    try {
        // Navigate with network event monitoring
        const responsePromise = page.waitForResponse(resp => 
            resp.url().includes('/api/products') && resp.status() === 200
        );
        
        await page.goto('https://example.com/products');
        const apiResponse = await responsePromise;
        
        // Parse API response directly
        const products = await apiResponse.json();
        console.log(`Found ${products.length} products from API`);
        
        // Alternative: Traditional DOM scraping
        await page.waitForSelector('.product-grid', { state: 'visible' });
        
        // Handle lazy-loaded images
        await page.evaluate(() => {
            const images = document.querySelectorAll('img[data-src]');
            images.forEach(img => {
                img.src = img.dataset.src;
            });
        });
        
        // Extract with auto-waiting
        const scrapedProducts = await page.locator('.product-card').evaluateAll(cards =>
            cards.map(card => ({
                title: card.querySelector('.title')?.textContent,
                price: parseFloat(card.querySelector('.price')?.textContent?.replace(/[^0-9.]/g, '')),
                availability: card.querySelector('.availability')?.textContent,
                rating: card.querySelector('.rating')?.getAttribute('data-rating')
            }))
        );
        
        // Handle pagination
        let hasNextPage = true;
        let pageNum = 1;
        
        while (hasNextPage && pageNum < 10) {
            const nextButton = page.locator('.pagination .next');
            hasNextPage = await nextButton.isVisible();
            
            if (hasNextPage) {
                await nextButton.click();
                await page.waitForLoadState('networkidle');
                pageNum++;
                
                // Scrape additional pages...
            }
        }
        
        return scrapedProducts;
        
    } finally {
        await context.close();
        await browser.close();
    }
}

// Mobile scraping example
async function mobileScraping() {
    const browser = await chromium.launch();
    const context = await browser.newContext({
        ...devices['iPhone 13 Pro Max']
    });
    
    const page = await context.newPage();
    await page.goto('https://m.example.com');
    
    // Mobile-specific selectors
    const mobileData = await page.locator('.mobile-product-list').evaluateAll(/*...*/);
    
    await browser.close();
    return mobileData;
}

# Python Example
from playwright.sync_api import sync_playwright
import json

def advanced_scraping():
    with sync_playwright() as p:
        # Launch browser with anti-detection measures
        browser = p.chromium.launch(
            headless=False,
            args=['--disable-blink-features=AutomationControlled']
        )
        
        # Create context with realistic settings
        context = browser.new_context(
            viewport={'width': 1920, 'height': 1080},
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            locale='en-US',
            timezone_id='America/New_York',
            permissions=['geolocation'],
            geolocation={'latitude': 40.7128, 'longitude': -74.0060}
        )
        
        # Block unnecessary resources
        context.route('**/*.{png,jpg,jpeg,gif,webp,svg,ico}', lambda route: route.abort())
        context.route('**/*.{css,font}', lambda route: route.abort())
        
        page = context.new_page()
        
        # Intercept and modify requests
        def handle_route(route, request):
            headers = {
                **request.headers,
                'Accept-Language': 'en-US,en;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'Cache-Control': 'no-cache',
                'Pragma': 'no-cache'
            }
            route.continue_(headers=headers)
        
        page.route('**/*', handle_route)
        
        try:
            # Navigate and wait for API response
            with page.expect_response(lambda resp: '/api/products' in resp.url and resp.status == 200) as response_info:
                page.goto('https://example.com/products')
            
            api_response = response_info.value
            products = api_response.json()
            print(f'Found {len(products)} products from API')
            
            # Wait for content to be visible
            page.wait_for_selector('.product-grid', state='visible')
            
            # Handle lazy-loaded images
            page.evaluate('''() => {
                const images = document.querySelectorAll('img[data-src]');
                images.forEach(img => {
                    img.src = img.dataset.src;
                });
            }''')
            
            # Extract data
            scraped_products = page.locator('.product-card').evaluate_all('''cards =>
                cards.map(card => ({
                    title: card.querySelector('.title')?.textContent,
                    price: parseFloat(card.querySelector('.price')?.textContent?.replace(/[^0-9.]/g, '')),
                    availability: card.querySelector('.availability')?.textContent,
                    rating: card.querySelector('.rating')?.getAttribute('data-rating')
                }))
            ''')
            
            # Handle pagination
            page_num = 1
            while page_num < 10:
                next_button = page.locator('.pagination .next')
                if next_button.is_visible():
                    next_button.click()
                    page.wait_for_load_state('networkidle')
                    page_num += 1
                    # Scrape additional pages...
                else:
                    break
            
            return scraped_products
            
        finally:
            context.close()
            browser.close()

# Mobile scraping example
def mobile_scraping():
    with sync_playwright() as p:
        browser = p.chromium.launch()
        context = browser.new_context(**p.devices['iPhone 13 Pro Max'])
        
        page = context.new_page()
        page.goto('https://m.example.com')
        
        # Mobile-specific scraping
        mobile_data = page.locator('.mobile-product-list').evaluate_all('...')
        
        browser.close()
        return mobile_data

// C# Example
using Microsoft.Playwright;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

class PlaywrightScraper
{
    public static async Task Main()
    {
        using var playwright = await Playwright.CreateAsync();
        
        // Launch browser with anti-detection
        await using var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions
        {
            Headless = false,
            Args = new[] { "--disable-blink-features=AutomationControlled" }
        });

        // Create context with realistic settings
        await using var context = await browser.NewContextAsync(new BrowserNewContextOptions
        {
            ViewportSize = new ViewportSize { Width = 1920, Height = 1080 },
            UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            Locale = "en-US",
            TimezoneId = "America/New_York",
            Permissions = new[] { "geolocation" },
            Geolocation = new Geolocation { Latitude = 40.7128f, Longitude = -74.0060f }
        });

        // Block unnecessary resources
        await context.RouteAsync("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", async route => await route.AbortAsync());
        await context.RouteAsync("**/*.{css,font}", async route => await route.AbortAsync());

        var page = await context.NewPageAsync();

        // Intercept requests
        await page.RouteAsync("**/*", async route =>
        {
            var headers = new Dictionary(route.Request.Headers)
            {
                ["Accept-Language"] = "en-US,en;q=0.9",
                ["Accept-Encoding"] = "gzip, deflate, br",
                ["Cache-Control"] = "no-cache",
                ["Pragma"] = "no-cache"
            };
            await route.ContinueAsync(new RouteContinueOptions { Headers = headers });
        });

        try
        {
            // Navigate and wait for response
            var responseTask = page.WaitForResponseAsync(resp => 
                resp.Url.Contains("/api/products") && resp.Status == 200);
            
            await page.GotoAsync("https://example.com/products");
            var apiResponse = await responseTask;
            
            var products = await apiResponse.JsonAsync>();
            Console.WriteLine($"Found {products.Count} products from API");

            // Wait for content
            await page.WaitForSelectorAsync(".product-grid", new PageWaitForSelectorOptions 
            { 
                State = WaitForSelectorState.Visible 
            });

            // Extract data
            var scrapedProducts = await page.Locator(".product-card").EvaluateAllAsync>(
                @"cards => cards.map(card => ({
                    title: card.querySelector('.title')?.textContent,
                    price: parseFloat(card.querySelector('.price')?.textContent?.replace(/[^0-9.]/g, '')),
                    availability: card.querySelector('.availability')?.textContent,
                    rating: card.querySelector('.rating')?.getAttribute('data-rating')
                }))"
            );

            // Handle pagination
            var hasNextPage = true;
            var pageNum = 1;

            while (hasNextPage && pageNum < 10)
            {
                var nextButton = page.Locator(".pagination .next");
                hasNextPage = await nextButton.IsVisibleAsync();

                if (hasNextPage)
                {
                    await nextButton.ClickAsync();
                    await page.WaitForLoadStateAsync(LoadState.NetworkIdle);
                    pageNum++;
                }
            }

            return scrapedProducts;
        }
        finally
        {
            await context.CloseAsync();
        }
    }
}

public class Product
{
    public string Title { get; set; }
    public decimal Price { get; set; }
    public string Availability { get; set; }
    public string Rating { get; set; }
}