import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import os

base_url = "https://blog.vacme.ch/hemed/"
visited = set()

def scrape_page(url):
    if url in visited:
        return
    visited.add(url)

    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()

        # Save HTML
        filename = url.replace(base_url, "").replace("/", "_") or "index"
        with open(f"vacme_blog/{filename}.html", "w", encoding="utf-8") as f:
            f.write(response.text)

        # Convert to Markdown (requires markdownify)
        from markdownify import markdownify as md
        with open(f"vacme_blog/{filename}.md", "w", encoding="utf-8") as f:
            f.write(md(response.text))

        # Find links recursively
        soup = BeautifulSoup(response.text, "html.parser")
        for link in soup.find_all("a", href=True):
            new_url = urljoin(base_url, link["href"])
            if new_url.startswith(base_url):
                scrape_page(new_url)

    except Exception as e:
        print(f"Error scraping {url}: {str(e)}")

# Create output directory
os.makedirs("vacme_blog", exist_ok=True)

# Start scraping
scrape_page(base_url)