Robots.txt Interpretation

# Parser robots.txt complet avec support des wildcards
import re
from dataclasses import dataclass
from typing import List, Optional, Dict
from urllib.parse import urlparse, urljoin
import httpx

@dataclass
class RobotsRule:
    path_pattern: str
    allowed: bool
    
    def matches(self, path: str) -> bool:
        """Vérifie si le chemin matche le pattern"""
        # Convertir le pattern robots.txt en regex
        regex_pattern = self.path_pattern
        regex_pattern = regex_pattern.replace('*', '.*')
        regex_pattern = regex_pattern.replace('$', '$')
        if not regex_pattern.endswith('$'):
            regex_pattern = f"^{regex_pattern}"
        else:
            regex_pattern = f"^{regex_pattern}"
        
        try:
            return bool(re.match(regex_pattern, path, re.IGNORECASE))
        except re.error:
            return self.path_pattern in path

@dataclass
class RobotsDirectives:
    """Directives pour un User-Agent spécifique"""
    user_agent: str
    rules: List[RobotsRule]
    crawl_delay: Optional[float] = None
    sitemaps: List[str] = None
    
    def __post_init__(self):
        if self.sitemaps is None:
            self.sitemaps = []

class AdvancedRobotsTxtParser:
    """Parser robots.txt avancé avec support complet"""
    
    def __init__(self):
        self.directives: Dict[str, RobotsDirectives] = {}
        self.sitemaps: List[str] = []
    
    def parse(self, content: str):
        """Parse le contenu du robots.txt"""
        current_agents: List[str] = []
        current_rules: List[RobotsRule] = []
        current_crawl_delay: Optional[float] = None
        
        for line in content.split('
'):
            line = line.split('#')[0].strip()  # Supprimer commentaires
            if not line:
                continue
            
            if ':' not in line:
                continue
            
            directive, value = line.split(':', 1)
            directive = directive.strip().lower()
            value = value.strip()
            
            if directive == 'user-agent':
                # Sauvegarder les règles précédentes
                if current_agents and current_rules:
                    for agent in current_agents:
                        self.directives[agent.lower()] = RobotsDirectives(
                            user_agent=agent,
                            rules=current_rules.copy(),
                            crawl_delay=current_crawl_delay,
                            sitemaps=self.sitemaps.copy()
                        )
                    current_rules = []
                    current_crawl_delay = None
                
                current_agents = [value]
                
            elif directive == 'disallow':
                if value:  # Disallow vide = tout autoriser
                    current_rules.append(RobotsRule(path_pattern=value, allowed=False))
                    
            elif directive == 'allow':
                current_rules.append(RobotsRule(path_pattern=value, allowed=True))
                
            elif directive == 'crawl-delay':
                try:
                    current_crawl_delay = float(value)
                except ValueError:
                    pass
                    
            elif directive == 'sitemap':
                self.sitemaps.append(value)
        
        # Sauvegarder les dernières règles
        if current_agents:
            for agent in current_agents:
                self.directives[agent.lower()] = RobotsDirectives(
                    user_agent=agent,
                    rules=current_rules,
                    crawl_delay=current_crawl_delay,
                    sitemaps=self.sitemaps
                )
    
    def can_fetch(self, user_agent: str, path: str) -> bool:
        """Détermine si l'URL peut être crawlée"""
        # Trouver les directives applicables
        agent_lower = user_agent.lower()
        directives = None
        
        # Chercher une correspondance exacte d'abord
        for key in self.directives:
            if key in agent_lower or agent_lower in key:
                directives = self.directives[key]
                break
        
        # Fallback sur * (wildcard)
        if not directives and '*' in self.directives:
            directives = self.directives['*']
        
        if not directives:
            return True  # Pas de règles = autorisé
        
        # Appliquer les règles (la plus spécifique gagne)
        matching_rules = [r for r in directives.rules if r.matches(path)]
        
        if not matching_rules:
            return True
        
        # Trier par longueur de pattern (plus long = plus spécifique)
        matching_rules.sort(key=lambda r: len(r.path_pattern), reverse=True)
        return matching_rules[0].allowed
    
    def get_crawl_delay(self, user_agent: str) -> Optional[float]:
        """Retourne le Crawl-delay pour un User-Agent"""
        agent_lower = user_agent.lower()
        
        for key, directives in self.directives.items():
            if key in agent_lower or agent_lower in key:
                return directives.crawl_delay
        
        if '*' in self.directives:
            return self.directives['*'].crawl_delay
        
        return None

# Exemple d'utilisation
async def check_robots(url: str, bot_name: str = "MyCrawler"):
    parsed = urlparse(url)
    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"
    
    async with httpx.AsyncClient() as client:
        response = await client.get(robots_url)
        
        parser = AdvancedRobotsTxtParser()
        parser.parse(response.text)
        
        can_crawl = parser.can_fetch(bot_name, parsed.path)
        delay = parser.get_crawl_delay(bot_name)
        
        return {
            'url': url,
            'can_crawl': can_crawl,
            'crawl_delay': delay,
            'sitemaps': parser.sitemaps
        }