test-html-cleaning.sh

#!/bin/bash

echo "Testing HTML cleaning in RAG preprocessing..."
echo "============================================="

# Clear existing chunks to force re-indexing
echo "1. Clearing existing chunks from database..."
psql -U akashswamy -d noteraity_vectors -c "DELETE FROM note_chunks;" > /dev/null 2>&1
echo "   ✅ Chunks cleared"

echo ""
echo "2. Testing HTML content processing..."

# Create a test HTML string similar to what the user showed
TEST_HTML='<p>My Bio: Joe Sweany</p><p>Age: 33</p><p>Location: St Nalbans, Watford, near London</p><hr><p>Asdfasdf&nbsp;</p>'

# Save it to a temporary file
echo "$TEST_HTML" > /tmp/test-html.html

echo "   Input HTML:"
echo "   $TEST_HTML"
echo ""

# The preprocessing should remove all HTML tags
echo "   Expected output (no HTML tags):"
echo "   My Bio: Joe Sweany Age: 33 Location: St Nalbans, Watford, near London Asdfasdf"
echo ""

echo "3. Instructions to test in the app:"
echo "   a. Start the app with: npm run tauri:dev"
echo "   b. Open or create a note with HTML content"
echo "   c. Save the note (triggers auto-indexing)"
echo "   d. Click the database icon in the note header"
echo "   e. Check the 'Chunks' tab in the debug modal"
echo "   f. Verify that chunks contain NO HTML tags"
echo ""

echo "4. What's been fixed:"
echo "   ✅ HTML entities (&nbsp;, &amp;, etc.) are now decoded"
echo "   ✅ All HTML tags are stripped properly"
echo "   ✅ Text is extracted recursively from nested elements"
echo "   ✅ Links are preserved in [url] format"
echo "   ✅ Double preprocessing issue fixed"
echo ""

echo "5. To verify the fix works, check console logs for:"
echo "   📝 Preprocessing note... (shows char count reduction)"
echo "   📦 Created X chunks for note..."
echo "   🔄 Generating embedding for chunk..."
echo "   ✅ Successfully generated embedding..."