Function fetch_endpoints

code-generator.py:78–132

Fetch endpoints from the provided Binance doc URLs, filtering duplicates.

()

def fetch_endpoints():
    """Fetch endpoints from the provided Binance doc URLs, filtering duplicates."""
    endpoints = set()
    deprecated_endpoints = set()
    for url in URLS:
        print(f"Fetching {url}")
        page = requests.get(url)
        soup = BeautifulSoup(page.content, "html.parser")
        all_code_blocks = soup.find_all("code")

        for code in all_code_blocks:
            code_text = code.get_text().strip()
            parts = code_text.split(" ")

            if len(parts) >= 2:
                parts[0] = parts[0].strip()
                parts[1] = parts[1].strip().split("?")[0]
            # Basic check for lines that look like: GET /path or POST /path etc.
            if (
                len(parts) >= 2
                and parts[0] in ["GET", "POST", "PUT", "DELETE"]
                and parts[1] is not None
                and parts[1] != ""
            ):
                method = parts[0]
                endpoint = parts[1]
                # Ensure endpoint starts with /
                if not endpoint.startswith("/"):
                    endpoint = "/" + endpoint

                # Check both deprecated prefixes and specific endpoints
                if (
                    any(endpoint.startswith(prefix) for prefix in DEPRECATED_PREFIXES)
                    or (method, endpoint) in DEPRECATED_ENDPOINTS
                ):
                    deprecated_endpoints.add((method, endpoint))
                    continue  # Skip adding to main endpoints set

                # Use a tuple of (method, endpoint) for uniqueness
                endpoints.add((method, endpoint))
    print(f"Found {len(deprecated_endpoints)} deprecated endpoints in the docs.")
    print(f"Found {len(endpoints)} unique endpoints in the docs.")

    # Filter endpoints that don't start with any known prefix
    valid_endpoints = {
        (method, endpoint)
        for method, endpoint in endpoints
        if any(endpoint.startswith(prefix) for prefix in PREFIX_MAP.keys())
    }

    filtered_count = len(endpoints) - len(valid_endpoints)
    print(f"Filtered out {filtered_count} endpoints that don't match known prefixes")
    print(f"Remaining endpoints: {len(valid_endpoints)}")

    return valid_endpoints


def get_request_function_and_path(