""" Parse Paradise Realty FLA area pages sitemap XML into a CSV. Output columns: - Community Name - Location - Meta Description - URL Link No UI buttons / no notebook display helpers. """ import re import sys import time from typing import List, Dict, Optional from urllib.parse import urlparse import requests import pandas as pd import xml.etree.ElementTree as ET SITEMAP_URL = "https://storage.googleapis.com/site-map-dynamic-page3/paradise_area_pages_sitemap.xml" OUT_CSV = "paradiserealtyfla-area-pages.csv" NS = { "sm": "http://www.sitemaps.org/schemas/sitemap/0.9", "image": "http://www.google.com/schemas/sitemap-image/1.1", } def status_bar(pct: int, prefix: str = "Progress") -> None: pct = max(0, min(100, int(pct))) bar_len = 30 filled = int(bar_len * pct / 100) bar = "█" * filled + "░" * (bar_len - filled) print(f"\r{prefix}: {pct:3d}% |{bar}|", end="", flush=True) if pct == 100: print() def fetch_xml(url: str) -> str: r = requests.get(url, timeout=30) r.raise_for_status() return r.text def slug_to_title(slug: str) -> str: # Convert URL slug to title-ish text slug = slug.replace("-", " ").strip() # Fix common abbreviations slug = re.sub(r"\bfl\b", "FL", slug, flags=re.I) slug = re.sub(r"\bst\b", "St.", slug, flags=re.I) slug = re.sub(r"\bpga\b", "PGA", slug, flags=re.I) # Title case but keep ALL-CAPS words words = [] for w in slug.split(): if w.isupper(): words.append(w) else: words.append(w.capitalize()) return " ".join(words) def parse_loc(loc: str) -> Dict[str, Optional[str]]: """ From a URL like: https://www.paradiserealtyfla.com/fields-lake-worth/ infer: Community Name = "Fields" Location = "Lake Worth" If pattern doesn't fit, Community Name becomes best-effort title from slug. """ path = urlparse(loc).path.strip("/") slug = path if path else "" parts = [p for p in slug.split("-") if p] community = None location = None if len(parts) >= 2: # Assume last 1-3 tokens are a city/county name # Heuristic: take last two if they look like location words # Otherwise last one. # We'll just take last two as location and rest as community. location_tokens = parts[-2:] community_tokens = parts[:-2] # If no community tokens (like /stuart-fl/), fallback to full slug if community_tokens: community = slug_to_title("-".join(community_tokens)) location = slug_to_title("-".join(location_tokens)) else: community = slug_to_title(slug) location = None else: community = slug_to_title(slug) if slug else loc location = None return {"Community Name": community, "Location": location} def parse_sitemap(xml_text: str) -> List[Dict[str, Optional[str]]]: root = ET.fromstring(xml_text) urls = root.findall("sm:url", NS) total = len(urls) rows: List[Dict[str, Optional[str]]] = [] status_bar(0, "Parsing XML") for i, url_el in enumerate(urls, start=1): loc_el = url_el.find("sm:loc", NS) loc = loc_el.text.strip() if loc_el is not None and loc_el.text else None caption_el = url_el.find("image:image/image:caption", NS) caption = caption_el.text.strip() if caption_el is not None and caption_el.text else None parsed = parse_loc(loc) if loc else {"Community Name": None, "Location": None} rows.append({ "Community Name": parsed["Community Name"], "Location": parsed["Location"], "Meta Description": caption, "URL Link": loc, }) pct = int(i * 100 / total) if total else 100 status_bar(pct, "Parsing XML") return rows def main(): try: xml_text = fetch_xml(SITEMAP_URL) except Exception as e: print("\nFailed to fetch sitemap:", e) sys.exit(1) rows = parse_sitemap(xml_text) df = pd.DataFrame(rows, columns=["Community Name", "Location", "Meta Description", "URL Link"]) df.to_csv(OUT_CSV, index=False) status_bar(100, "Writing CSV") print(f"Saved: {OUT_CSV} ({len(df)} rows)") if __name__ == "__main__": main()