Advanced Concurrency

ThreadPoolExecutor, ProcessPoolExecutor, and concurrency patterns.

Overview

Master advanced concurrency patterns.

ThreadPoolExecutor

from concurrent.futures import ThreadPoolExecutor, as_completed
import requests

def fetch_url(url):
    response = requests.get(url)
    return response.status_code

urls = [
    "https://httpbin.org/delay/1",
    "https://httpbin.org/delay/2",
    "https://httpbin.org/delay/3",
]

with ThreadPoolExecutor(max_workers=5) as executor:
    futures = {executor.submit(fetch_url, url): url for url in urls}
    
    for future in as_completed(futures):
        url = futures[future]
        try:
            result = future.result()
            print(f"{url}: {result}")
        except Exception as e:
            print(f"{url} failed: {e}")

ProcessPoolExecutor

from concurrent.futures import ProcessPoolExecutor
import os

def cpu_bound_task(n):
    return sum(i * i for i in range(n))

if __name__ == "__main__":
    numbers = [10**6, 10**7, 10**8]
    
    with ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
        results = list(executor.map(cpu_bound_task, numbers))
    
    print(results)

Async Patterns

import asyncio
import aiohttp

async def fetch(session, url):
    async with session.get(url) as response:
        return await response.text()

async def main():
    urls = [
        "https://httpbin.org/delay/1",
        "https://httpbin.org/delay/2",
    ]
    
    async with aiohttp.ClientSession() as session:
        tasks = [fetch(session, url) for url in urls]
        results = await asyncio.gather(*tasks)
        print(results)

asyncio.run(main())

Practice

Implement a parallel web crawler with rate limiting.