zumiez_analyzer-grok3.py

#!/usr/bin/env python3
# Requirements: requests, beautifulsoup4, selenium, webdriver-manager, fake-useragent
# Install with:
#   pip install requests beautifulsoup4 selenium webdriver-manager fake-useragent

import os
import re
import json
import time
import logging
import random
import requests
import datetime
import string
import uuid
import shutil
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent
from selenium.common.exceptions import TimeoutException, WebDriverException

# Set up logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

def safe_write_file(filename, content, mode='w'):
    """Write content to file with permission error handling"""
    try:
        # Try using the current directory
        with open(filename, mode, encoding='utf-8') as f:
            f.write(content)
        logging.info(f"Successfully wrote to file: {filename}")
        return True
    except (IOError, PermissionError) as e:
        logging.error(f"Permission error writing to {filename}: {e}")
        
        # Try using /tmp as fallback
        try:
            tmp_filename = os.path.join('/tmp', os.path.basename(filename))
            with open(tmp_filename, mode, encoding='utf-8') as f:
                f.write(content)
            logging.info(f"Wrote to alternate location: {tmp_filename}")
            
            # If successful, try to copy back to original location
            try:
                shutil.copy(tmp_filename, filename)
                logging.info(f"Copied from {tmp_filename} to {filename}")
                return True
            except Exception as copy_error:
                logging.error(f"Couldn't copy from temp to original: {copy_error}")
                return False
        except Exception as tmp_error:
            logging.error(f"Could not write to temp location either: {tmp_error}")
            return False

def create_chrome_temp_dir():
    """Create a properly permissioned temporary directory for Chrome"""
    # Try using the system TMPDIR environment variable
    system_tmpdir = os.environ.get('TMPDIR', '/tmp')
    unique_id = str(uuid.uuid4())
    temp_dir = os.path.join(system_tmpdir, f'chrome_data_{unique_id}')
    
    try:
        # Create directory with full permissions
        os.makedirs(temp_dir, mode=0o777, exist_ok=True)
        os.chmod(temp_dir, 0o777)  # Ensure permissions are set correctly
        logging.info(f"Created Chrome temp dir: {temp_dir}")
        return temp_dir
    except Exception as e:
        logging.error(f"Failed to create Chrome temp dir: {e}")
        
        # Fallback to using tempfile module
        try:
            import tempfile
            alt_temp_dir = tempfile.mkdtemp(prefix='chrome_data_')
            os.chmod(alt_temp_dir, 0o777)
            logging.info(f"Created alternate temp dir: {alt_temp_dir}")
            return alt_temp_dir
        except Exception as alt_e:
            logging.error(f"Failed to create alternate temp dir: {alt_e}")
            return None

def fetch_page(url, max_retries=3, timeout=30):
    ua = UserAgent()
    
    for attempt in range(max_retries):
        user_agent = ua.random
        logging.info(f"Using user agent: {user_agent}")

        # Create a properly permissioned temporary directory
        temp_dir = create_chrome_temp_dir()
        if not temp_dir:
            logging.error("Could not create temp directory, skipping attempt")
            time.sleep(2)
            continue

        options = Options()
        
        # Set headless mode for CI environment
        if os.getenv("CI", "false").lower() == "true":
            options.add_argument("--headless=new")
            logging.info("Running in headless mode for CI")
        
        # Essential Chrome options
        options.add_argument(f"user-agent={user_agent}")
        options.add_argument(f"--user-data-dir={temp_dir}")
        options.add_argument("--disable-extensions")
        options.add_argument("--disable-notifications")
        options.add_argument("--disable-web-security")
        options.add_argument("--allow-running-insecure-content")
        options.add_argument("--disable-blink-features=AutomationControlled")
        options.add_argument("--disable-infobars")
        options.add_argument("--window-size=1920,1080")
        options.add_argument("--no-first-run")
        options.add_argument("--no-default-browser-check")
        options.add_argument("--disable-popup-blocking")
        options.add_experimental_option("excludeSwitches", ["enable-automation"])
        options.add_experimental_option("useAutomationExtension", False)
        
        # Add CI-specific options
        if os.getenv("CI"):
            options.add_argument("--no-sandbox")
            options.add_argument("--disable-dev-shm-usage")
            options.add_argument("--disable-gpu")
            options.add_argument("--remote-debugging-port=0")
            logging.info("Added CI-specific options")

        # Use webdriver-manager to get the Chromedriver path
        try:
            chromedriver_path = ChromeDriverManager().install()
            service = Service(executable_path=chromedriver_path)
            logging.info(f"Using chromedriver at {service.path}")
        except Exception as e:
            logging.error(f"Failed to install ChromeDriver: {e}")
            if attempt < max_retries - 1:
                time.sleep(2 ** attempt)
                continue
            else:
                return None

        driver = None
        try:
            logging.info(f"Fetching {url} (Attempt {attempt + 1})")
            
            # Initialize WebDriver
            driver_attempts = 3
            for driver_attempt in range(driver_attempts):
                try:
                    logging.info(f"Initializing WebDriver (Attempt {driver_attempt + 1})")
                    driver = webdriver.Chrome(service=service, options=options)
                    logging.info("WebDriver initialized successfully")
                    break
                except TimeoutException as e:
                    logging.error(f"TimeoutException during WebDriver init: {e}")
                    if driver_attempt < driver_attempts - 1:
                        time.sleep(5)
                        continue
                    else:
                        raise
                except WebDriverException as e:
                    logging.error(f"WebDriverException during WebDriver init: {e}")
                    
                    # Handle specific error cases
                    if "user data directory is already in use" in str(e):
                        logging.warning("User data directory issue, creating a fresh one")
                        # Try to clean up the directory
                        try:
                            if temp_dir and os.path.exists(temp_dir):
                                shutil.rmtree(temp_dir)
                            temp_dir = create_chrome_temp_dir()
                            options.add_argument(f"--user-data-dir={temp_dir}")
                        except Exception as cleanup_error:
                            logging.error(f"Failed to clean up user data dir: {cleanup_error}")
                    
                    if "cannot find Chrome binary" in str(e):
                        logging.error("Ensure Chrome is correctly installed and in the system's PATH")
                        
                    if driver_attempt < driver_attempts - 1:
                        time.sleep(5)
                        continue
                    else:
                        raise

            if not driver:
                logging.error("Failed to initialize WebDriver after multiple attempts")
                continue

            # Set page load timeout
            driver.set_page_load_timeout(timeout)
            
            # Introduce randomized delay before loading page
            time.sleep(random.uniform(1, 3))
            
            # Navigate to the URL
            driver.get(url)
            
            # Wait for page to be fully loaded
            WebDriverWait(driver, timeout).until(
                lambda d: d.execute_script("return document.readyState") == "complete"
            )

            # Additional wait for any JavaScript to finish
            time.sleep(random.uniform(3, 5))
            logging.info("Initial wait for dynamic content")

            # Check if we've been redirected to an undesired page
            current_url = driver.current_url
            if "stash" in current_url.lower():
                logging.error("Redirected to Stash page, retrying")
                driver.quit()
                continue

            # Wait for product elements to appear
            try:
                WebDriverWait(driver, 15).until(
                    EC.presence_of_element_located((By.CSS_SELECTOR, "li.ProductCard, .product-card, .product-item, a[href*='deck'], a[href*='wheels'], a[href*='truck'], a[href*='bearings']"))
                )
                logging.info("Product listings detected")
            except Exception as e:
                logging.warning(f"Could not detect product listings: {e}")

            # Infinite scroll implementation
            logging.info("Attempting infinite scroll")
            max_scroll_attempts = 8
            scroll_attempts = 0
            previous_item_count = 0

            while scroll_attempts < max_scroll_attempts:
                # Scroll to bottom
                driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
                time.sleep(random.uniform(2, 4))
                
                # Count items
                current_items = len(driver.find_elements(By.CSS_SELECTOR, "li.ProductCard, .product-card, .product-item, a[href*='deck'], a[href*='wheels'], a[href*='truck'], a[href*='bearings']"))
                logging.info(f"Scroll attempt {scroll_attempts + 1}: found {current_items} items")

                # Check for redirects
                current_url = driver.current_url
                if "stash" in current_url.lower():
                    logging.error("Redirected to Stash page during scrolling")
                    driver.quit()
                    return None

                # If no new items loaded, we've reached the end
                if current_items == previous_item_count and current_items > 0:
                    logging.info("No more items to load")
                    break

                previous_item_count = current_items
                scroll_attempts += 1

            # Final wait for any AJAX requests to complete
            time.sleep(random.uniform(2, 4))
            logging.info("Final wait for AJAX content")

            # Scroll back to top
            driver.execute_script("window.scrollTo(0, 0);")
            time.sleep(random.uniform(1, 2))

            # Get the page source
            html = driver.page_source
            logging.info(f"Successfully fetched {url}")
            return html

        except Exception as e:
            logging.error(f"Failed to fetch {url}: {e}")
            if attempt < max_retries - 1:
                time.sleep(2 ** attempt + random.uniform(2, 5))
            else:
                logging.error(f"Max retries reached for {url}")
                return None
        finally:
            # Always ensure WebDriver is closed properly
            if driver:
                try:
                    driver.quit()
                    logging.info("WebDriver closed successfully")
                except Exception as e:
                    logging.warning(f"Error quitting driver: {e}")
            
            # Clean up the temp directory
            try:
                if temp_dir and os.path.exists(temp_dir):
                    shutil.rmtree(temp_dir)
                    logging.info(f"Removed temp directory: {temp_dir}")
            except Exception as e:
                logging.warning(f"Failed to remove temp directory: {e}")

def save_debug_file(filename, content):
    """Safely save debug files with permission error handling."""
    safe_write_file(filename, content)

class Scraper:
    def __init__(self, name, url, part):
        self.name = name
        self.url = url
        self.part = part

    def scrape(self):
        html = fetch_page(self.url)
        return self.parse(html)

    def parse(self, html):
        raise NotImplementedError

class ZumiezScraper(Scraper):
    def parse(self, html):
        if not html:
            logging.error("No HTML to parse")
            return []

        soup = BeautifulSoup(html, "html.parser")
        products = []
        seen = set()

        save_debug_file(f"zumiez_debug_{self.part.lower()}.html", html)
        product_grid = soup.select("li.ProductCard")
        logging.info(f"Found {len(product_grid)} product containers")

        for product in product_grid:
            try:
                link = product.select_one("a.ProductCard-Link")
                if not link:
                    logging.warning("No link found for product")
                    continue
                href = link["href"]
                if href.startswith("/"):
                    href = "https://www.zumiez.com" + href
                if href in seen:
                    logging.info(f"Duplicate URL skipped: {href}")
                    continue
                seen.add(href)

                name_el = product.select_one(".ProductCard-Name")
                name = name_el.get_text(strip=True) if name_el else link.find("img", alt=True).get("alt", "").strip()
                if not name:
                    logging.warning(f"No name found for {href}")
                    continue

                if self.part == "Wheels":
                    if not any(brand in name for brand in ["Bones", "Powell", "Spitfire", "OJ"]):
                        logging.info(f"Skipping product not from Bones, Powell, Spitfire, or OJ: {name}")
                        continue

                sale_price_el = product.select_one(".ProductPrice-PriceValue")
                original_price_el = product.select_one(".ProductCardPrice-HighPrice")
                sale_price = sale_price_el.get_text(strip=True).replace("$", "") if sale_price_el else None
                original_price = original_price_el.get_text(strip=True).replace("$", "") if original_price_el else None

                if not sale_price:
                    logging.warning(f"No sale price found for {href}")
                    continue

                # Log discount percentage for decks
                if self.part == "Decks":
                    percent_off = calculate_percent_off(sale_price, original_price)
                    logging.info(f"Deck {name}: {percent_off} off")
                    try:
                        percent_off_value = float(percent_off.strip("%"))
                        if percent_off_value < 30:
                            logging.info(f"Skipping deck with less than 30% off: {name} ({percent_off})")
                            continue
                    except (ValueError, TypeError):
                        logging.info(f"Skipping deck with invalid % off: {name} ({percent_off})")
                        continue

                availability = "Check store"
                products.append({
                    "name": name,
                    "url": href,
                    "price_new": sale_price,
                    "price_old": original_price,
                    "availability": availability,
                    "part": self.part
                })
                logging.info(f"Parsed product: {name}")

            except Exception as e:
                logging.error(f"Error parsing product: {e}")
                continue

        logging.info(f"Parsed {len(products)} products")
        return products

class SkateWarehouseScraper(Scraper):
    def parse(self, html):
        if not html:
            logging.error("No HTML to parse")
            return []

        soup = BeautifulSoup(html, "html.parser")
        products = []
        seen = set()

        save_debug_file(f"skatewarehouse_debug_{self.part.lower()}.html", html)

        for a in soup.find_all("a", href=True):
            text = a.get_text(strip=True)
            href = a["href"]

            # Skip non-product links (relaxed filter)
            if not any(part in href.lower() for part in ["wheels", "truck", "bearings", "deck"]) and not any(brand.lower() in href.lower() for brand in ["bones", "spitfire", "independent", "bronson"]):
                continue

            # Filter based on part type
            if self.part == "Wheels" and "Wheels" not in text:
                continue
            if self.part == "Trucks" and "Truck" not in text:
                continue
            if self.part == "Bearings" and "Bearings" not in text:
                continue
            if self.part == "Decks" and "Deck" not in text:
                continue

            if href.startswith("/"):
                href = "https://www.skatewarehouse.com" + href
            if href in seen:
                logging.info(f"Duplicate URL skipped: {href}")
                continue

            prices = re.findall(r"\$(\d+\.\d{2})", text)
            if not prices:
                continue

            name = text.split(f"${prices[0]}")[0].strip()
            if not name:
                logging.warning(f"No name found for {href}")
                continue

            if self.part == "Wheels":
                if not any(brand in name for brand in ["Bones", "Powell", "Spitfire", "OJ"]):
                    logging.info(f"Skipping product not from Bones, Powell, Spitfire, or OJ: {name}")
                    continue
            elif self.part == "Trucks":
                if not any(brand in name for brand in ["Independent", "Indy", "Ace"]):
                    logging.info(f"Skipping product not from Independent or Ace Trucks: {name}")
                    continue
            elif self.part == "Decks":
                # Calculate % off and filter for 30%+ discount
                price_new = prices[0]
                price_old = prices[1] if len(prices) > 1 else None
                percent_off = calculate_percent_off(price_new, price_old)
                logging.info(f"Deck {name}: {percent_off} off")
                try:
                    percent_off_value = float(percent_off.strip("%"))
                    if percent_off_value < 30:
                        logging.info(f"Skipping deck with less than 30% off: {name} ({percent_off})")
                        continue
                except (ValueError, TypeError):
                    logging.info(f"Skipping deck with invalid % off: {name} ({percent_off})")
                    continue

            seen.add(href)
            price_old = prices[1] if len(prices) > 1 else None
            products.append({
                "name": name,
                "url": href,
                "price_new": prices[0],
                "price_old": price_old,
                "availability": "Check store",
                "part": self.part
            })
            logging.info(f"Parsed product: {name}")

        logging.info(f"Parsed {len(products)} products")
        return products

# CCS Scraper Fix
class CCSScraper(Scraper):
    def parse(self, html):
        if not html:
            logging.error("No HTML to parse")
            return []

        soup = BeautifulSoup(html, "html.parser")
        products = []
        seen = set()

        save_debug_file(f"ccs_debug_{self.part.lower()}.html", html)

        # Updated selectors for CCS website
        # The site appears to use a product-card structure based on search results
        for prod in soup.select(".product-card, .product-item, .product"):
            try:
                # Try multiple possible selectors for different elements
                title_el = prod.select_one(".product-title, .product-name, h3")
                price_el = prod.select_one(".product-price--sale, .product-price, .price--sale")
                compare_price_el = prod.select_one(".product-price--compare, .compare-at-price, .price--compare")
                link_el = prod.select_one("a")
                
                if not (title_el and price_el and link_el):
                    logging.warning("Missing title, price, or link for product")
                    continue

                name = title_el.get_text(strip=True)
                if not name:
                    logging.warning("No name found for product")
                    continue

                href = link_el.get("href")
                if href.startswith("/"):
                    href = "https://shop.ccs.com" + href
                if href in seen:
                    logging.info(f"Duplicate URL skipped: {href}")
                    continue
                seen.add(href)

                # Extract prices using regex to handle different formats
                price_text = price_el.get_text(strip=True)
                prices = re.findall(r"\$(\d+\.\d{2})", price_text)
                if not prices:
                    logging.warning(f"No prices found for {href}")
                    continue
                
                price_new = prices[0]
                
                # Try to get original price from compare price element
                price_old = None
                if compare_price_el:
                    compare_text = compare_price_el.get_text(strip=True)
                    compare_prices = re.findall(r"\$(\d+\.\d{2})", compare_text)
                    if compare_prices:
                        price_old = compare_prices[0]

                # For decks, filter for 30%+ discount
                if self.part == "Decks":
                    percent_off = calculate_percent_off(price_new, price_old)
                    logging.info(f"Deck {name}: {percent_off} off")
                    try:
                        percent_off_value = float(percent_off.strip("%"))
                        if percent_off_value < 30:
                            logging.info(f"Skipping deck with less than 30% off: {name} ({percent_off})")
                            continue
                    except (ValueError, TypeError):
                        logging.info(f"Skipping deck with invalid % off: {name} ({percent_off})")
                        continue

                if self.part == "Wheels":
                    if not any(brand in name for brand in ["Bones", "Powell", "Spitfire", "OJ"]):
                        logging.info(f"Skipping product not from Bones, Powell, Spitfire, or OJ: {name}")
                        continue

                products.append({
                    "name": name,
                    "url": href,
                    "price_new": price_new,
                    "price_old": price_old,
                    "availability": "Check store",
                    "part": self.part
                })
                logging.info(f"Parsed product: {name}")

            except Exception as e:
                logging.error(f"Error parsing product: {e}")
                continue

        logging.info(f"Parsed {len(products)} products")
        return products

class ZumiezDecksScraper(ZumiezScraper):
    def __init__(self):
        super().__init__("Zumiez", "https://www.zumiez.com/skate/skateboard-decks.html?customFilters=promotion_flag:Sale", "Decks")

# Tactics Decks Scraper Fix
class TacticsDecksScraper(Scraper):
    def __init__(self):
        super().__init__("Tactics", "https://www.tactics.com/skateboard-decks/sale", "Decks")

    def parse(self, html):
        if not html:
            logging.error("No HTML to parse")
            return []

        soup = BeautifulSoup(html, "html.parser")
        products = []
        seen = set()

        save_debug_file(f"tactics_debug_decks.html", html)

        # Updated selectors for current Tactics website
        product_containers = soup.select(".product-card, .product-item, article.product, .product")
        logging.info(f"Found {len(product_containers)} product containers")

        if len(product_containers) == 0:
            # Try alternative selectors if the main ones don't work
            product_containers = soup.select("[itemtype*='Product']")
            logging.info(f"Using fallback selector, found {len(product_containers)} product containers")

        for container in product_containers:
            try:
                # Try multiple possible selectors
                link = container.select_one("a[href]")
                if not link:
                    logging.warning("No link found for product")
                    continue
                
                href = link["href"]
                if href.startswith("/"):
                    href = "https://www.tactics.com" + href
                if href in seen:
                    logging.info(f"Duplicate URL skipped: {href}")
                    continue
                seen.add(href)

                # Try multiple possible selectors for the name
                name_el = container.select_one(".product-card__title, .product-name, h3, [itemprop='name']")
                name = name_el.get_text(strip=True) if name_el else ""
                
                # Try getting name from title attribute if not found
                if not name and link.get("title"):
                    name = link.get("title", "").strip()
                    
                if not name:
                    logging.warning(f"No name found for {href}")
                    continue

                # Look for price elements with multiple possible selectors
                price_new_el = container.select_one(".sale-price, .product-price--sale, .price.sale, [itemprop='price']")
                price_old_el = container.select_one(".compare-price, .product-price--compare, .price.compare, [itemprop='comparePrice']")
                
                # Try to extract prices using regex
                price_new = None
                if price_new_el:
                    price_text = price_new_el.get_text(strip=True)
                    price_matches = re.findall(r"\$?(\d+\.\d{2})", price_text)
                    if price_matches:
                        price_new = price_matches[0]
                
                price_old = None
                if price_old_el:
                    price_old_text = price_old_el.get_text(strip=True)
                    price_old_matches = re.findall(r"\$?(\d+\.\d{2})", price_old_text)
                    if price_old_matches:
                        price_old = price_old_matches[0]
                
                # If we still don't have prices, look for any price text in the container
                if not price_new:
                    all_text = container.get_text(strip=True)
                    all_prices = re.findall(r"\$(\d+\.\d{2})", all_text)
                    if len(all_prices) >= 2:
                        # Assume first price is sale price, second is original
                        price_new = all_prices[0]
                        price_old = all_prices[1]
                    elif len(all_prices) == 1:
                        price_new = all_prices[0]

                if not price_new:
                    logging.warning(f"No prices found for {href}")
                    continue

                # Calculate % off and filter for 30%+ discount
                percent_off = calculate_percent_off(price_new, price_old)
                logging.info(f"Deck {name}: {percent_off} off")
                try:
                    percent_off_value = float(percent_off.strip("%"))
                    if percent_off_value < 30:
                        logging.info(f"Skipping deck with less than 30% off: {name} ({percent_off})")
                        continue
                except (ValueError, TypeError):
                    logging.info(f"Skipping deck with invalid % off: {name} ({percent_off})")
                    continue

                products.append({
                    "name": name,
                    "url": href,
                    "price_new": price_new,
                    "price_old": price_old,
                    "availability": "Check store",
                    "part": self.part
                })
                logging.info(f"Parsed product: {name}")

            except Exception as e:
                logging.error(f"Error parsing product: {e}")
                continue

        logging.info(f"Parsed {len(products)} products")
        return products

def load_previous(path="previous_data.json"):
    """Load previous data with permission error handling."""
    try:
        if os.path.exists(path):
            with open(path, 'r') as f:
                return json.load(f)
        return {}
    except Exception as e:
        logging.error(f"Error loading previous data: {e}")
        return {}

def save_current(data, path="previous_data.json"):
    """Save current data with permission error handling."""
    try:
        return safe_write_file(path, json.dumps(data, indent=2))
    except Exception as e:
        logging.error(f"Error saving current data: {e}")
        return False

def compare(prev, curr):
    changes = {}
    for site, items in curr.items():
        prev_map = {i["url"]: i for i in prev.get(site, [])}
        diffs = []
        for it in items:
            pi = prev_map.get(it["url"])
            if not pi:
                diffs.append({"type": "new", "item": it})
            elif it["price_new"] != pi.get("price_new"):
                diffs.append({
                    "type": "price_change",
                    "url": it["url"],
                    "old": pi.get("price_new"),
                    "new": it["price_new"],
                    "name": it["name"]
                })
        curr_urls = {i["url"] for i in items}
        for url, pi in prev_map.items():
            if url not in curr_urls:
                diffs.append({"type": "removed", "item": pi})
        if diffs:
            changes[site] = diffs
    return changes

def calculate_percent_off(price_new, price_old):
    try:
        new = float(price_new)
        old = float(price_old)
        if old <= 0:
            return "N/A"
        percent_off = ((old - new) / old) * 100
        return f"{percent_off:.2f}%"
    except (ValueError, TypeError):
        return "N/A"

def generate_html_chart(data, changes, output_file="sale_items_chart.html"):
    """
    Generate an improved HTML chart with enhanced historical changes section.
    Enhancements: add part type and date to changes, show change summary in headers, enable sorting.
    """
    # Get the current date and time for historical changes and title
    current_date = datetime.datetime.now().strftime("%Y-%m-%d")
    current_datetime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

    html_content = f"""
    <!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>Sale Items and Changes</title>
        <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;500;700&display=swap" rel="stylesheet">
        <style>
            body {{
                font-family: 'Roboto', sans-serif;
                margin: 20px;
                background-color: #f5f7fa;
                color: #333;
            }}
            h1, h2, h3 {{
                text-align: center;
                color: #2c3e50;
            }}
            h1 {{
                font-size: 2.2em;
                margin-bottom: 20px;
            }}
            h2 {{
                font-size: 1.8em;
                margin-top: 40px;
            }}
            h3 {{
                font-size: 1.4em;
                margin: 20px 0;
                cursor: pointer;
                display: flex;
                align-items: center;
                gap: 8px;
            }}
            h3::before {{
                content: '▼';
                font-size: 0.8em;
                transition: transform 0.3s;
            }}
            h3.collapsed::before {{
                content: '▶';
                transform: rotate(0deg);
            }}
            .summary {{
                background-color: #ffffff;
                padding: 15px;
                border-radius: 8px;
                box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
                margin-bottom: 20px;
                text-align: center;
            }}
            .search-container {{
                margin: 20px 0;
                text-align: center;
            }}
            .search-container input {{
                padding: 10px;
                width: 300px;
                border: 1px solid #ddd;
                border-radius: 5px;
                font-size: 1em;
            }}
            table {{
                width: 100%;
                border-collapse: collapse;
                margin: 20px 0;
                background-color: #ffffff;
                box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
                border-radius: 8px;
                overflow: hidden;
            }}
            th, td {{
                padding: 12px;
                text-align: left;
                border-bottom: 1px solid #e0e0e0;
            }}
            th {{
                background: linear-gradient(135deg, #3498db, #2980b9);
                color: white;
                position: sticky;
                top: 0;
                z-index: 1;
                cursor: pointer;
                font-weight: 500;
            }}
            th:hover {{
                background: linear-gradient(135deg, #2980b9, #1c5f8a);
            }}
            th::after {{
                content: '';
                margin-left: 5px;
                font-size: 0.8em;
            }}
            th.asc::after {{
                content: '↑';
            }}
            th.desc::after {{
                content: '↓';
            }}
            tr:nth-child(even) {{
                background-color: #f9f9f9;
            }}
            tr:hover {{
                background-color: #f1f1f1;
            }}
            a {{
                color: #3498db;
                text-decoration: none;
            }}
            a:hover {{
                text-decoration: underline;
            }}
            .new {{
                background-color: #e6f7e6;
            }}
            .price-change {{
                background-color: #fff4e1;
            }}
            .removed {{
                background-color: #ffe6e6;
            }}
            .section {{
                margin-bottom: 40px;
            }}
            .collapsible-content {{
                display: block;
                transition: max-height 0.3s ease-out;
                overflow: hidden;
            }}
            .collapsible-content.collapsed {{
                display: none;
            }}
            @media (max-width: 768px) {{
                table {{
                    display: block;
                    overflow-x: auto;
                }}
                th, td {{
                    min-width: 120px;
                }}
            }}
        </style>
        <script>
            function sortTable(tableId, colIndex, isNumeric = false) {{
                const table = document.getElementById(tableId);
                let rows = Array.from(table.rows).slice(1);
                const isAsc = table.rows[0].cells[colIndex].getAttribute('data-sort') !== 'asc';
                
                rows.sort((a, b) => {{
                    let aValue = a.cells[colIndex].innerText.trim();
                    let bValue = b.cells[colIndex].innerText.trim();
                    
                    if (isNumeric) {{
                        aValue = parseFloat(aValue.replace('$', '').replace('%', '')) || 0;
                        bValue = parseFloat(bValue.replace('$', '').replace('%', '')) || 0;
                        return isAsc ? aValue - bValue : bValue - aValue;
                    }} else {{
                        return isAsc ? aValue.localeCompare(bValue) : bValue.localeCompare(aValue);
                    }}
                }});
                
                table.rows[0].cells[colIndex].setAttribute('data-sort', isAsc ? 'asc' : 'desc');
                for (let i = 0; i < table.rows[0].cells.length; i++) {{
                    table.rows[0].cells[i].classList.remove('asc', 'desc');
                }}
                table.rows[0].cells[colIndex].classList.add(isAsc ? 'asc' : 'desc');
                
                const tbody = table.getElementsByTagName('tbody')[0];
                tbody.innerHTML = '';
                rows.forEach(row => tbody.appendChild(row));
            }}

            function searchTable() {{
                const input = document.getElementById('searchInput').value.toLowerCase();
                const tables = document.getElementsByTagName('table');
                
                for (let table of tables) {{
                    const rows = table.getElementsByTagName('tr');
                    for (let i = 1; i < rows.length; i++) {{
                        const cells = rows[i].getElementsByTagName('td');
                        let match = false;
                        for (let cell of cells) {{
                            if (cell.innerText.toLowerCase().includes(input)) {{
                                match = true;
                                break;
                            }}
                        }}
                        rows[i].style.display = match ? '' : 'none';
                    }}
                }}
            }}

            function toggleSection(element) {{
                const content = element.nextElementSibling;
                element.classList.toggle('collapsed');
                content.classList.toggle('collapsed');
            }}
        </script>
    </head>
    <body>
        <h1>Skateboard Sale Items and Changes as of {current_datetime}</h1>
    """

    # Summary Statistics
    summary = {}
    for site_key, items in data.items():
        store, part = site_key.split("_")
        if store not in summary:
            summary[store] = {}
        summary[store][part] = len(items)

    html_content += "<div class='summary'><h2>Summary</h2>"
    for store, parts in summary.items():
        total = sum(parts.values())
        parts_str = ", ".join([f"{count} {part}" for part, count in parts.items()])
        html_content += f"<p><strong>{store}</strong>: {total} items ({parts_str})</p>"
    html_content += "</div>"

    # Search Bar
    html_content += """
        <div class="search-container">
            <input type="text" id="searchInput" onkeyup="searchTable()" placeholder="Search items...">
        </div>
    """

    # Current Sale Items - Grouped by Store
    html_content += "<div class='section'><h2>Current Sale Items</h2>"
    stores = sorted(set(site_key.split("_")[0] for site_key in data.keys()))
    for store in stores:
        store_items = []
        for site_key, items in data.items():
            if site_key.startswith(store):
                store_items.extend(items)
        if not store_items:
            continue

        html_content += f"<h3 onclick='toggleSection(this)'>{store}</h3>"
        html_content += f"<div class='collapsible-content'>"
        html_content += f"<table id='table-{store.lower()}'>"
        html_content += """
            <thead>
                <tr>
                    <th onclick="sortTable('table-{}', 0)">Part</th>
                    <th onclick="sortTable('table-{}', 1)">Product Name</th>
                    <th onclick="sortTable('table-{}', 2, true)">New Price ($)</th>
                    <th onclick="sortTable('table-{}', 3, true)">Old Price ($)</th>
                    <th onclick="sortTable('table-{}', 4, true)">% Off</th>
                    <th onclick="sortTable('table-{}', 5)">Availability</th>
                </tr>
            </thead>
            <tbody>
        """.format(store.lower(), store.lower(), store.lower(), store.lower(), store.lower(), store.lower())

        for item in store_items:
            percent_off = calculate_percent_off(item["price_new"], item["price_old"])
            html_content += f"""
                <tr>
                    <td>{item['part']}</td>
                    <td><a href="{item['url']}" target="_blank">{item['name']}</a></td>
                    <td>{item['price_new']}</td>
                    <td>{item['price_old'] if item['price_old'] else 'N/A'}</td>
                    <td>{percent_off}</td>
                    <td>{item['availability']}</td>
                </tr>
            """
        html_content += "</tbody></table></div>"
    html_content += "</div>"

    # Historical Changes with Enhanced Details
    html_content += """
        <div class="section">
            <h2>Historical Changes</h2>
    """
    if not changes:
        html_content += "<p>No changes detected since the last run.</p>"
    else:
        for site, site_changes in changes.items():
            # Calculate summary of changes
            new_count = len([c for c in site_changes if c["type"] == "new"])
            price_change_count = len([c for c in site_changes if c["type"] == "price_change"])
            removed_count = len([c for c in site_changes if c["type"] == "removed"])
            summary_parts = []
            if new_count > 0:
                summary_parts.append(f"{new_count} New")
            if price_change_count > 0:
                summary_parts.append(f"{price_change_count} Price Changes")
            if removed_count > 0:
                summary_parts.append(f"{removed_count} Removed")
            summary_text = ", ".join(summary_parts) if summary_parts else "No changes"

            html_content += f"<h3 onclick='toggleSection(this)'>{site} ({summary_text})</h3>"
            html_content += "<div class='collapsible-content'>"

            # New Items
            new_items = [change for change in site_changes if change["type"] == "new"]
            if new_items:
                html_content += f"""
                <h4>New Items</h4>
                <table id="historical-new-{site.lower().replace('_', '-')}">
                    <thead>
                        <tr>
                            <th onclick="sortTable('historical-new-{site.lower().replace('_', '-')}', 0)">Part</th>
                            <th onclick="sortTable('historical-new-{site.lower().replace('_', '-')}', 1)">Product Name</th>
                            <th onclick="sortTable('historical-new-{site.lower().replace('_', '-')}', 2, true)">New Price ($)</th>
                            <th onclick="sortTable('historical-new-{site.lower().replace('_', '-')}', 3)">Date Added</th>
                        </tr>
                    </thead>
                    <tbody>
                """
                for change in new_items:
                    item = change["item"]
                    html_content += f"""
                        <tr class="new">
                            <td>{item['part']}</td>
                            <td><a href="{item['url']}" target="_blank">{item['name']}</a></td>
                            <td>{item['price_new']}</td>
                            <td>{current_date}</td>
                        </tr>
                    """
                html_content += "</tbody></table>"

            # Price Changes
            price_changes = [change for change in site_changes if change["type"] == "price_change"]
            if price_changes:
                html_content += f"""
                <h4>Price Changes</h4>
                <table id="historical-price-{site.lower().replace('_', '-')}">
                    <thead>
                        <tr>
                            <th onclick="sortTable('historical-price-{site.lower().replace('_', '-')}', 0)">Part</th>
                            <th onclick="sortTable('historical-price-{site.lower().replace('_', '-')}', 1)">Product Name</th>
                            <th onclick="sortTable('historical-price-{site.lower().replace('_', '-')}', 2, true)">Old Price ($)</th>
                            <th onclick="sortTable('historical-price-{site.lower().replace('_', '-')}', 3, true)">New Price ($)</th>
                            <th onclick="sortTable('historical-price-{site.lower().replace('_', '-')}', 4)">Date Changed</th>
                        </tr>
                    </thead>
                    <tbody>
                """
                for change in price_changes:
                    part = site.split("_")[1]
                    html_content += f"""
                        <tr class="price-change">
                            <td>{part}</td>
                            <td><a href="{change['url']}" target="_blank">{change['name']}</a></td>
                            <td>{change['old']}</td>
                            <td>{change['new']}</td>
                            <td>{current_date}</td>
                        </tr>
                    """
                html_content += "</tbody></table>"

            # Removed Items
            removed_items = [change for change in site_changes if change["type"] == "removed"]
            if removed_items:
                html_content += f"""
                <h4>Removed Items</h4>
                <table id="historical-removed-{site.lower().replace('_', '-')}">
                    <thead>
                        <tr>
                            <th onclick="sortTable('historical-removed-{site.lower().replace('_', '-')}', 0)">Part</th>
                            <th onclick="sortTable('historical-removed-{site.lower().replace('_', '-')}', 1)">Product Name</th>
                            <th onclick="sortTable('historical-removed-{site.lower().replace('_', '-')}', 2, true)">Last Known Price ($)</th>
                            <th onclick="sortTable('historical-removed-{site.lower().replace('_', '-')}', 3)">Date Removed</th>
                        </tr>
                    </thead>
                    <tbody>
                """
                for change in removed_items:
                    item = change["item"]
                    html_content += f"""
                        <tr class="removed">
                            <td>{item['part']}</td>
                            <td>{item['name']}</td>
                            <td>{item['price_new']}</td>
                            <td>{current_date}</td>
                        </tr>
                    """
                html_content += "</tbody></table>"

            html_content += "</div>"

    html_content += """
        </div>
    </body>
    </html>
    """

    # Save with permission error handling
    if safe_write_file(output_file, html_content):
        logging.info(f"Generated HTML chart at {output_file}")
    else:
        logging.error(f"Failed to write HTML chart to {output_file}")

def main():
    try:
        scrapers = [
            ZumiezScraper("Zumiez", "https://www.zumiez.com/skate/components/wheels.html?customFilters=brand:Bones,OJ%20Wheels,Powell,Spitfire;promotion_flag:Sale", "Wheels"),
            SkateWarehouseScraper("SkateWarehouse", "https://www.skatewarehouse.com/searchresults.html?filter_cat=SALEWHEELS&filter_type=Wheels#filter_cat=SALEWHEELS&filter_type=Wheels&brand_str%5B%5D=Bones%20Wheels&brand_str%5B%5D=Spitfire&opt_page=1&opt_sort=alphaAtoZ&opt_perpage=20", "Wheels"),
            CCSScraper("CCS", "https://shop.ccs.com/collections/clearance/skateboard-wheels", "Wheels"),
            ZumiezScraper("Zumiez", "https://www.zumiez.com/skate/components/trucks.html?customFilters=brand:Ace%20Trucks,Independent;promotion_flag:Sale", "Trucks"),
            SkateWarehouseScraper("SkateWarehouse", "https://www.skatewarehouse.com/Clearance_Skateboard_Trucks/catpage-SALETRUCKS.html", "Trucks"),
            SkateWarehouseScraper("SkateWarehouse", "https://www.skatewarehouse.com/Clearance_Skateboard_Parts/catpage-BOXLSHOES.html", "Bearings"),
            ZumiezDecksScraper(),
            SkateWarehouseScraper("SkateWarehouse", "https://www.skatewarehouse.com/Clearance_Skateboard_Decks/catpage-SALEDECK.html", "Decks"),
            CCSScraper("CCS", "https://shop.ccs.com/collections/clearance/skateboard-deck", "Decks"),
            TacticsDecksScraper(),
        ]

        current = {}
        for s in scrapers:
            site_key = f"{s.name}_{s.part}"
            logging.info(f"Scraping {site_key}")
            current[site_key] = s.scrape()
            logging.info(f"Finished scraping {site_key}: {len(current[site_key])} items")

        for site, items in current.items():
            print(f"{site}: {len(items)} items scraped")

        combined_data = {}
        for site_key, items in current.items():
            store, part = site_key.split("_")
            combined_items = []
            for item in items:
                item["store"] = store
            combined_items.extend(items)
            combined_data[site_key] = combined_items

        previous = load_previous()
        diffs = compare(previous, combined_data)
        if diffs:
            print("Changes detected:")
            for site, changes in diffs.items():
                print(f"\n{site}:")
                for c in changes:
                    if c["type"] == "new":
                        print(f"  New: {c['item']['name']} at {c['item']['price_new']}")
                    elif c["type"] == "price_change":
                        print(f"  Price change: {c['old']} -> {c['new']} | {c['url']}")
                    elif c["type"] == "removed":
                        print(f"  Removed: {c['item']['name']}")
        else:
            print("No changes detected.")

        generate_html_chart(combined_data, diffs)
        save_current(combined_data)
        
    except Exception as e:
        logging.error(f"Error in main function: {e}")
        # Try to save what we have
        if 'combined_data' in locals() and combined_data:
            logging.info("Attempting to save partial data...")
            save_current(combined_data)

if __name__ == "__main__":
    main()