exclude_paths

exclude_paths: Optional[List[str]] = Field(
  default=None,
  description='Regex patterns to exclude URLs from the crawl with specific path patterns.\n\n        Use when the user explicitly asks to exclude a specific path from a website.\n\n        ex. "Crawl example.com but exclude the /api/v1 path form the crawl" ---> ["/api/v1.*"] \n        ex. "Crawl example.com but exclude the /documentation path from the crawl" ---> ["/documentation/.*"]\n        '
)