feat(articles): added content scoring and readability assessment features

* Implemented new Scorer type for heuristic scoring of HTML content based on readability principles.

* Developed confidence calculation method to assess the reliability of content scores.

internal/articles/heuristics.go

···
+package articles
+
+import (
+	"strings"
+
+	"github.com/antchfx/htmlquery"
+	"golang.org/x/net/html"
+)
+
+// ExtractionResult contains the results of heuristic content extraction.
+type ExtractionResult struct {
+	Content          string
+	Title            string
+	Author           string
+	PublishedDate    string
+	SiteName         string
+	Language         string
+	Confidence       float64
+	ExtractionMethod string // "heuristic" or "xpath" or "dual"
+}
+
+// HeuristicExtractor implements Readability-style content extraction.
+type HeuristicExtractor struct {
+	scorer *Scorer
+}
+
+// NewHeuristicExtractor creates a new extractor with default scoring.
+func NewHeuristicExtractor() *HeuristicExtractor {
+	return &HeuristicExtractor{
+		scorer: NewScorer(),
+	}
+}
+
+// ExtractContent performs heuristic-based content extraction from an HTML document.
+func (e *HeuristicExtractor) ExtractContent(doc *html.Node) *ExtractionResult {
+	if doc == nil {
+		return nil
+	}
+
+	if !e.scorer.IsProbablyReadable(doc) {
+		return &ExtractionResult{
+			Confidence:       0.1,
+			ExtractionMethod: "heuristic",
+		}
+	}
+
+	cleaned := e.cleanDocument(doc)
+	candidates := e.scorer.FindTopCandidates(cleaned, 5)
+	if len(candidates) == 0 {
+		return &ExtractionResult{
+			Confidence:       0.2,
+			ExtractionMethod: "heuristic",
+		}
+	}
+
+	topCandidate := candidates[0]
+	content := e.extractTextContent(topCandidate.Node)
+	result := &ExtractionResult{
+		Content:          content,
+		Confidence:       topCandidate.ConfidenceLevel,
+		ExtractionMethod: "heuristic",
+	}
+
+	return result
+}
+
+// cleanDocument removes unwanted elements and prepares the document for extraction.
+func (e *HeuristicExtractor) cleanDocument(doc *html.Node) *html.Node {
+
+	cloned := e.cloneNode(doc)
+
+	e.removeElements(cloned, "script", "style", "noscript", "iframe", "embed", "object")
+	e.removeHiddenElements(cloned)
+	e.removeUnlikelyCandidates(cloned)
+	e.removeHighLinkDensityElements(cloned)
+
+	return cloned
+}
+
+// cloneNode creates a deep copy of an HTML node tree.
+func (e *HeuristicExtractor) cloneNode(node *html.Node) *html.Node {
+	if node == nil {
+		return nil
+	}
+
+	clone := &html.Node{
+		Type:      node.Type,
+		Data:      node.Data,
+		DataAtom:  node.DataAtom,
+		Namespace: node.Namespace,
+		Attr:      make([]html.Attribute, len(node.Attr)),
+	}
+
+	copy(clone.Attr, node.Attr)
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		clonedChild := e.cloneNode(child)
+		if clonedChild != nil {
+			clone.AppendChild(clonedChild)
+		}
+	}
+
+	return clone
+}
+
+// removeElements removes all elements with the specified tag names.
+func (e *HeuristicExtractor) removeElements(root *html.Node, tagNames ...string) {
+	if root == nil {
+		return
+	}
+
+	tagMap := make(map[string]bool)
+	for _, tag := range tagNames {
+		tagMap[strings.ToLower(tag)] = true
+	}
+
+	var toRemove []*html.Node
+
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			if tagMap[strings.ToLower(node.Data)] {
+				toRemove = append(toRemove, node)
+				return
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+
+	walk(root)
+
+	for _, node := range toRemove {
+		if node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+// removeHiddenElements removes elements that are hidden via CSS or attributes.
+func (e *HeuristicExtractor) removeHiddenElements(root *html.Node) {
+	if root == nil {
+		return
+	}
+
+	var toRemove []*html.Node
+
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			for _, attr := range node.Attr {
+				if attr.Key == "hidden" {
+					toRemove = append(toRemove, node)
+					return
+				}
+
+				if attr.Key == "style" {
+					style := strings.ToLower(attr.Val)
+					if strings.Contains(style, "display:none") || strings.Contains(style, "display: none") ||
+						strings.Contains(style, "visibility:hidden") || strings.Contains(style, "visibility: hidden") {
+						toRemove = append(toRemove, node)
+						return
+					}
+				}
+
+				if attr.Key == "aria-hidden" && attr.Val == "true" {
+					toRemove = append(toRemove, node)
+					return
+				}
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+
+	walk(root)
+
+	for _, node := range toRemove {
+		if node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+// removeUnlikelyCandidates removes elements that are unlikely to be main content.
+func (e *HeuristicExtractor) removeUnlikelyCandidates(root *html.Node) {
+	if root == nil {
+		return
+	}
+
+	var toRemove []*html.Node
+
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			score := e.scorer.getClassIdScore(node)
+
+			if score < -40 {
+				toRemove = append(toRemove, node)
+				return
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+
+	walk(root)
+
+	for _, node := range toRemove {
+		if node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+// removeHighLinkDensityElements removes elements with excessive link density.
+func (e *HeuristicExtractor) removeHighLinkDensityElements(root *html.Node) {
+	if root == nil {
+		return
+	}
+
+	const linkDensityThreshold = 0.75
+
+	var toRemove []*html.Node
+
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			if strings.ToLower(node.Data) == "a" {
+				for child := node.FirstChild; child != nil; child = child.NextSibling {
+					walk(child)
+				}
+				return
+			}
+
+			density := e.scorer.calculateLinkDensity(node)
+			textLen := e.scorer.calculateTextLength(node)
+
+			if density > linkDensityThreshold && textLen < 500 {
+				toRemove = append(toRemove, node)
+				return
+			}
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			walk(child)
+		}
+	}
+
+	walk(root)
+
+	for _, node := range toRemove {
+		if node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+// extractTextContent extracts cleaned text from a node.
+func (e *HeuristicExtractor) extractTextContent(node *html.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	var buf strings.Builder
+	e.extractTextRecursive(node, &buf)
+
+	text := buf.String()
+	text = normalizeWhitespace(text)
+	text = strings.TrimSpace(text)
+
+	return text
+}
+
+// extractTextRecursive recursively extracts text with basic formatting.
+func (e *HeuristicExtractor) extractTextRecursive(node *html.Node, buf *strings.Builder) {
+	if node == nil {
+		return
+	}
+
+	if node.Type == html.TextNode {
+		buf.WriteString(node.Data)
+		return
+	}
+
+	if node.Type == html.ElementNode {
+		tag := strings.ToLower(node.Data)
+
+		if e.isBlockElement(tag) && buf.Len() > 0 {
+			buf.WriteString("\n\n")
+		}
+
+		if tag == "li" {
+			buf.WriteString("\n• ")
+		}
+
+		for child := node.FirstChild; child != nil; child = child.NextSibling {
+			e.extractTextRecursive(child, buf)
+		}
+
+		if e.isBlockElement(tag) {
+			buf.WriteString("\n")
+		}
+	}
+}
+
+// isBlockElement returns true for block-level HTML elements.
+func (e *HeuristicExtractor) isBlockElement(tagName string) bool {
+	blockElements := map[string]bool{
+		"p":          true,
+		"div":        true,
+		"article":    true,
+		"section":    true,
+		"h1":         true,
+		"h2":         true,
+		"h3":         true,
+		"h4":         true,
+		"h5":         true,
+		"h6":         true,
+		"blockquote": true,
+		"pre":        true,
+		"ul":         true,
+		"ol":         true,
+		"table":      true,
+		"tr":         true,
+		"td":         true,
+		"th":         true,
+	}
+
+	return blockElements[tagName]
+}
+
+// CompareWithXPath compares heuristic extraction with XPath-based extraction.
+func (e *HeuristicExtractor) CompareWithXPath(doc *html.Node, xpathNode *html.Node) *ExtractionResult {
+	if doc == nil {
+		return nil
+	}
+
+	heuristicResult := e.ExtractContent(doc)
+	if heuristicResult == nil {
+		heuristicResult = &ExtractionResult{
+			ExtractionMethod: "heuristic",
+			Confidence:       0.0,
+		}
+	}
+
+	if xpathNode == nil {
+		return heuristicResult
+	}
+
+	xpathContent := e.extractTextContent(xpathNode)
+	xpathLen := len(xpathContent)
+	heuristicLen := len(heuristicResult.Content)
+
+	similarity := e.calculateSimilarity(xpathContent, heuristicResult.Content)
+
+	if similarity > 0.8 {
+		heuristicResult.Confidence = 0.95
+		heuristicResult.ExtractionMethod = "dual-validated"
+		return heuristicResult
+	} else if float64(xpathLen) > float64(heuristicLen)*1.5 {
+		return &ExtractionResult{
+			Content:          xpathContent,
+			Confidence:       0.85,
+			ExtractionMethod: "xpath-preferred",
+		}
+	} else if float64(heuristicLen) > float64(xpathLen)*1.5 {
+		heuristicResult.Confidence = 0.80
+		heuristicResult.ExtractionMethod = "heuristic-preferred"
+		return heuristicResult
+	} else {
+		heuristicResult.Confidence = 0.70
+		heuristicResult.ExtractionMethod = "heuristic-fallback"
+		return heuristicResult
+	}
+}
+
+// calculateSimilarity estimates content similarity (simple ratio of common words).
+func (e *HeuristicExtractor) calculateSimilarity(text1, text2 string) float64 {
+	if len(text1) == 0 || len(text2) == 0 {
+		if len(text1) == 0 && len(text2) == 0 {
+			return 1.0
+		}
+		return 0.0
+	}
+
+	words1 := strings.Fields(strings.ToLower(text1))
+	words2 := strings.Fields(strings.ToLower(text2))
+
+	if len(words1) == 0 || len(words2) == 0 {
+		return 0.0
+	}
+
+	freq1 := make(map[string]int)
+	freq2 := make(map[string]int)
+
+	for _, word := range words1 {
+		freq1[word]++
+	}
+
+	for _, word := range words2 {
+		freq2[word]++
+	}
+
+	common := 0
+	for word := range freq1 {
+		if freq2[word] > 0 {
+			common++
+		}
+	}
+
+	union := len(freq1) + len(freq2) - common
+	if union == 0 {
+		return 0.0
+	}
+
+	return float64(common) / float64(union)
+}
+
+// ExtractWithSemanticHTML attempts extraction using semantic HTML5 elements first.
+// Falls back to heuristic scoring if semantic elements aren't found.
+func (e *HeuristicExtractor) ExtractWithSemanticHTML(doc *html.Node) *ExtractionResult {
+	if doc == nil {
+		return nil
+	}
+
+	articleNode := htmlquery.FindOne(doc, "//article")
+	if articleNode != nil {
+		content := e.extractTextContent(articleNode)
+		if len(content) > e.scorer.minContentLength {
+			return &ExtractionResult{
+				Content:          content,
+				Confidence:       0.90,
+				ExtractionMethod: "semantic-html",
+			}
+		}
+	}
+
+	mainNode := htmlquery.FindOne(doc, "//main")
+	if mainNode != nil {
+		content := e.extractTextContent(mainNode)
+		if len(content) > e.scorer.minContentLength {
+			return &ExtractionResult{
+				Content:          content,
+				Confidence:       0.88,
+				ExtractionMethod: "semantic-html",
+			}
+		}
+	}
+
+	return e.ExtractContent(doc)
+}

internal/articles/heuristics_test.go

···
+package articles
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func TestHeuristicExtractor(t *testing.T) {
+	t.Run("NewHeuristicExtractor", func(t *testing.T) {
+		t.Run("creates extractor with scorer", func(t *testing.T) {
+			extractor := NewHeuristicExtractor()
+
+			if extractor == nil {
+				t.Fatal("Expected extractor to be created, got nil")
+			}
+
+			if extractor.scorer == nil {
+				t.Error("Expected extractor to have scorer")
+			}
+		})
+	})
+
+	t.Run("ExtractContent", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("extracts content from article", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<article class="main-content">
+					<p>This is the first paragraph of the article with substantial content.</p>
+					<p>This is the second paragraph with more information and details.</p>
+					<p>And this is the third paragraph to ensure sufficient content.</p>
+				</article>
+				<aside class="sidebar"><a href="#">Sidebar link</a></aside>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractContent(doc)
+
+			if result == nil {
+				t.Fatal("Expected extraction result, got nil")
+			}
+
+			if result.Content == "" {
+				t.Error("Expected content to be extracted")
+			}
+
+			if result.Confidence == 0.0 {
+				t.Error("Expected non-zero confidence")
+			}
+
+			if !strings.Contains(result.Content, "first paragraph") {
+				t.Error("Expected content to contain article text")
+			}
+		})
+
+		t.Run("returns low confidence for unreadable document", func(t *testing.T) {
+			htmlStr := `<html><body><div>Short</div></body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractContent(doc)
+
+			if result == nil {
+				t.Fatal("Expected extraction result, got nil")
+			}
+
+			if result.Confidence > 0.3 {
+				t.Errorf("Expected low confidence for short document, got %f", result.Confidence)
+			}
+		})
+
+		t.Run("returns nil for nil document", func(t *testing.T) {
+			result := extractor.ExtractContent(nil)
+
+			if result != nil {
+				t.Error("Expected nil for nil document")
+			}
+		})
+	})
+
+	t.Run("cleanDocument", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("removes script and style tags", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<script>alert('test');</script>
+				<style>.test { color: red; }</style>
+				<p>Content</p>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			cleaned := extractor.cleanDocument(doc)
+
+			script := findElement(cleaned, "script")
+			style := findElement(cleaned, "style")
+
+			if script != nil {
+				t.Error("Expected script tag to be removed")
+			}
+
+			if style != nil {
+				t.Error("Expected style tag to be removed")
+			}
+		})
+
+		t.Run("removes hidden elements", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<div style="display:none">Hidden</div>
+				<div hidden>Also hidden</div>
+				<p>Visible</p>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			cleaned := extractor.cleanDocument(doc)
+
+			// Count divs - should only have visible ones
+			divCount := 0
+			var countDivs func(*html.Node)
+			countDivs = func(node *html.Node) {
+				if node.Type == html.ElementNode && node.Data == "div" {
+					divCount++
+				}
+				for child := node.FirstChild; child != nil; child = child.NextSibling {
+					countDivs(child)
+				}
+			}
+			countDivs(cleaned)
+
+			if divCount > 0 {
+				t.Errorf("Expected hidden divs to be removed, found %d", divCount)
+			}
+		})
+
+		t.Run("removes high link density elements", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<div class="links">
+					<a href="#">Link1</a>
+					<a href="#">Link2</a>
+					<a href="#">Link3</a>
+				</div>
+				<p>Regular paragraph with actual content that should remain.</p>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			cleaned := extractor.cleanDocument(doc)
+
+			p := findElement(cleaned, "p")
+			if p == nil {
+				t.Error("Expected paragraph to remain")
+			}
+		})
+	})
+
+	t.Run("extractTextContent", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("extracts text with basic formatting", func(t *testing.T) {
+			htmlStr := `<html><body><div>
+				<p>First paragraph</p>
+				<p>Second paragraph</p>
+			</div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			text := extractor.extractTextContent(div)
+
+			if !strings.Contains(text, "First paragraph") {
+				t.Error("Expected text to contain first paragraph")
+			}
+
+			if !strings.Contains(text, "Second paragraph") {
+				t.Error("Expected text to contain second paragraph")
+			}
+		})
+
+		t.Run("formats list items with bullets", func(t *testing.T) {
+			htmlStr := `<html><body><ul>
+				<li>Item 1</li>
+				<li>Item 2</li>
+			</ul></body></html>`
+			doc := parseHTML(htmlStr)
+			ul := findElement(doc, "ul")
+
+			text := extractor.extractTextContent(ul)
+
+			if !strings.Contains(text, "•") {
+				t.Error("Expected text to contain bullet points")
+			}
+		})
+
+		t.Run("returns empty string for nil node", func(t *testing.T) {
+			text := extractor.extractTextContent(nil)
+
+			if text != "" {
+				t.Error("Expected empty string for nil node")
+			}
+		})
+	})
+
+	t.Run("CompareWithXPath", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("high confidence when XPath and heuristics agree", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<article>
+					<p>This is substantial content that both methods should find.</p>
+					<p>Another paragraph with more details and information.</p>
+					<p>And a third paragraph for good measure and completeness.</p>
+				</article>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+			article := findElement(doc, "article")
+
+			result := extractor.CompareWithXPath(doc, article)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if result.Confidence < 0.8 {
+				t.Errorf("Expected high confidence when methods agree, got %f", result.Confidence)
+			}
+
+			if !strings.Contains(result.ExtractionMethod, "dual") && !strings.Contains(result.ExtractionMethod, "validated") {
+				t.Errorf("Expected dual validation method, got %s", result.ExtractionMethod)
+			}
+		})
+
+		t.Run("prefers XPath when it extracts more content", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<div class="content">
+					<p>Short content</p>
+				</div>
+				<div class="more">
+					<p>This is additional content that XPath found but heuristics might miss.</p>
+					<p>Even more content here to make a significant difference in length.</p>
+					<p>And yet another paragraph to ensure XPath extraction is substantially longer.</p>
+				</div>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			// XPath would get more content
+			body := findElement(doc, "body")
+
+			result := extractor.CompareWithXPath(doc, body)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			// Should prefer one method over the other
+			if result.ExtractionMethod == "heuristic" {
+				t.Errorf("Expected method preference, got %s", result.ExtractionMethod)
+			}
+		})
+
+		t.Run("uses heuristics when XPath node is nil", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<article>
+					<p>Content that heuristics should find on its own.</p>
+					<p>Additional paragraph for sufficient content length.</p>
+					<p>Third paragraph to meet minimum requirements.</p>
+				</article>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.CompareWithXPath(doc, nil)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if result.ExtractionMethod != "heuristic" {
+				t.Errorf("Expected heuristic method when XPath is nil, got %s", result.ExtractionMethod)
+			}
+		})
+
+		t.Run("returns nil for nil document", func(t *testing.T) {
+			result := extractor.CompareWithXPath(nil, nil)
+
+			if result != nil {
+				t.Error("Expected nil for nil document")
+			}
+		})
+	})
+
+	t.Run("calculateSimilarity", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("returns high similarity for identical text", func(t *testing.T) {
+			text := "This is some test content"
+
+			similarity := extractor.calculateSimilarity(text, text)
+
+			if similarity < 0.9 {
+				t.Errorf("Expected high similarity for identical text, got %f", similarity)
+			}
+		})
+
+		t.Run("returns low similarity for different text", func(t *testing.T) {
+			text1 := "This is the first piece of content"
+			text2 := "Completely different words and phrases"
+
+			similarity := extractor.calculateSimilarity(text1, text2)
+
+			if similarity > 0.3 {
+				t.Errorf("Expected low similarity for different text, got %f", similarity)
+			}
+		})
+
+		t.Run("returns zero for empty strings", func(t *testing.T) {
+			similarity := extractor.calculateSimilarity("text", "")
+
+			if similarity != 0.0 {
+				t.Errorf("Expected zero similarity for empty string, got %f", similarity)
+			}
+		})
+
+		t.Run("returns one for both empty", func(t *testing.T) {
+			similarity := extractor.calculateSimilarity("", "")
+
+			if similarity != 1.0 {
+				t.Errorf("Expected 1.0 similarity for both empty, got %f", similarity)
+			}
+		})
+	})
+
+	t.Run("ExtractWithSemanticHTML", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("extracts from article tag", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<nav>Navigation</nav>
+				<article>
+					<p>This is the main article content that should be extracted.</p>
+					<p>Second paragraph of the article with more information.</p>
+					<p>Third paragraph to provide sufficient content length.</p>
+				</article>
+				<aside>Sidebar</aside>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractWithSemanticHTML(doc)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if result.ExtractionMethod != "semantic-html" {
+				t.Errorf("Expected semantic-html method, got %s", result.ExtractionMethod)
+			}
+
+			if !strings.Contains(result.Content, "main article content") {
+				t.Error("Expected content from article tag")
+			}
+
+			if result.Confidence < 0.85 {
+				t.Errorf("Expected high confidence for semantic HTML, got %f", result.Confidence)
+			}
+		})
+
+		t.Run("extracts from main tag", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<header>Header</header>
+				<main>
+					<p>This is the main content area with sufficient text.</p>
+					<p>Additional content paragraph with more details.</p>
+					<p>Third paragraph for completeness and length.</p>
+				</main>
+				<footer>Footer</footer>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractWithSemanticHTML(doc)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if result.ExtractionMethod != "semantic-html" {
+				t.Errorf("Expected semantic-html method, got %s", result.ExtractionMethod)
+			}
+
+			if !strings.Contains(result.Content, "main content area") {
+				t.Error("Expected content from main tag")
+			}
+		})
+
+		t.Run("falls back to heuristics without semantic tags", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<div class="content">
+					<p>Content in a regular div without semantic HTML tags.</p>
+					<p>Second paragraph with additional information.</p>
+					<p>Third paragraph for sufficient content.</p>
+				</div>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractWithSemanticHTML(doc)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if result.ExtractionMethod == "semantic-html" {
+				t.Error("Should not use semantic-html method without semantic tags")
+			}
+		})
+
+		t.Run("returns nil for nil document", func(t *testing.T) {
+			result := extractor.ExtractWithSemanticHTML(nil)
+
+			if result != nil {
+				t.Error("Expected nil for nil document")
+			}
+		})
+	})
+
+	t.Run("isBlockElement", func(t *testing.T) {
+		extractor := NewHeuristicExtractor()
+
+		t.Run("identifies block elements", func(t *testing.T) {
+			blockTags := []string{"p", "div", "article", "h1", "section"}
+
+			for _, tag := range blockTags {
+				if !extractor.isBlockElement(tag) {
+					t.Errorf("Expected %s to be a block element", tag)
+				}
+			}
+		})
+
+		t.Run("identifies non-block elements", func(t *testing.T) {
+			inlineTags := []string{"span", "a", "em", "strong", "code"}
+
+			for _, tag := range inlineTags {
+				if extractor.isBlockElement(tag) {
+					t.Errorf("Expected %s to not be a block element", tag)
+				}
+			}
+		})
+	})
+}

internal/articles/metadata.go

···
+package articles
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/antchfx/htmlquery"
+	"golang.org/x/net/html"
+)
+
+// MetadataExtractor implements multi-strategy metadata extraction from HTML documents.
+// It attempts to extract article metadata using OpenGraph, Schema.org, meta tags,
+// and semantic HTML5 elements, with fallback chains for each field.
+type MetadataExtractor struct{}
+
+// NewMetadataExtractor creates a new metadata extractor.
+func NewMetadataExtractor() *MetadataExtractor {
+	return &MetadataExtractor{}
+}
+
+// ExtractMetadata extracts all available metadata from an HTML document.
+// Returns an ExtractionResult with populated metadata fields.
+func (m *MetadataExtractor) ExtractMetadata(doc *html.Node) *ExtractionResult {
+	if doc == nil {
+		return &ExtractionResult{}
+	}
+
+	result := &ExtractionResult{}
+
+	result.Title = m.ExtractTitle(doc)
+	result.Author = m.ExtractAuthor(doc)
+	result.PublishedDate = m.ExtractPublishedDate(doc)
+	result.SiteName = m.ExtractSiteName(doc)
+	result.Language = m.ExtractLanguage(doc)
+
+	return result
+}
+
+// ExtractTitle extracts the article title using multiple strategies.
+// Tries in order: OpenGraph, Schema.org, meta tags, h1, title tag.
+func (m *MetadataExtractor) ExtractTitle(doc *html.Node) string {
+	if doc == nil {
+		return ""
+	}
+
+	if title := m.getMetaContent(doc, "property", "og:title"); title != "" {
+		return title
+	}
+
+	if title := m.getSchemaOrgField(doc, "headline"); title != "" {
+		return title
+	}
+
+	if title := m.getSchemaOrgField(doc, "name"); title != "" {
+		return title
+	}
+
+	if title := m.getMetaContent(doc, "name", "twitter:title"); title != "" {
+		return title
+	}
+
+	if title := m.getMetaContent(doc, "property", "article:title"); title != "" {
+		return title
+	}
+
+	if h1 := htmlquery.FindOne(doc, "//h1"); h1 != nil {
+		if title := htmlquery.InnerText(h1); title != "" {
+			return strings.TrimSpace(title)
+		}
+	}
+
+	if titleNode := htmlquery.FindOne(doc, "//title"); titleNode != nil {
+		if title := htmlquery.InnerText(titleNode); title != "" {
+			return strings.TrimSpace(title)
+		}
+	}
+
+	return ""
+}
+
+// ExtractAuthor extracts the article author using multiple strategies.
+// Tries in order: OpenGraph, Schema.org, meta tags, rel=author, byline elements.
+func (m *MetadataExtractor) ExtractAuthor(doc *html.Node) string {
+	if doc == nil {
+		return ""
+	}
+
+	if author := m.getMetaContent(doc, "property", "og:author"); author != "" {
+		return author
+	}
+
+	if author := m.getSchemaOrgField(doc, "author"); author != "" {
+		return author
+	}
+
+	if author := m.getMetaContent(doc, "property", "article:author"); author != "" {
+		return author
+	}
+
+	if author := m.getMetaContent(doc, "name", "twitter:creator"); author != "" {
+		return author
+	}
+
+	if author := m.getMetaContent(doc, "name", "author"); author != "" {
+		return author
+	}
+
+	if authorLink := htmlquery.FindOne(doc, "//a[@rel='author']"); authorLink != nil {
+		if author := htmlquery.InnerText(authorLink); author != "" {
+			return strings.TrimSpace(author)
+		}
+	}
+
+	bylineSelectors := []string{
+		"//span[contains(@class, 'author')]",
+		"//div[contains(@class, 'author')]",
+		"//p[contains(@class, 'byline')]",
+		"//span[contains(@class, 'byline')]",
+	}
+
+	for _, selector := range bylineSelectors {
+		if node := htmlquery.FindOne(doc, selector); node != nil {
+			if author := htmlquery.InnerText(node); author != "" {
+				return strings.TrimSpace(author)
+			}
+		}
+	}
+
+	return ""
+}
+
+// ExtractPublishedDate extracts the publication date using multiple strategies.
+// Tries in order: OpenGraph, Schema.org, article:published_time, time elements.
+func (m *MetadataExtractor) ExtractPublishedDate(doc *html.Node) string {
+	if doc == nil {
+		return ""
+	}
+
+	if date := m.getMetaContent(doc, "property", "og:published_time"); date != "" {
+		return date
+	}
+
+	if date := m.getSchemaOrgField(doc, "datePublished"); date != "" {
+		return date
+	}
+
+	if date := m.getSchemaOrgField(doc, "publishDate"); date != "" {
+		return date
+	}
+
+	if date := m.getMetaContent(doc, "property", "article:published_time"); date != "" {
+		return date
+	}
+
+	if date := m.getMetaContent(doc, "name", "publication_date"); date != "" {
+		return date
+	}
+
+	if date := m.getMetaContent(doc, "name", "date"); date != "" {
+		return date
+	}
+
+	if timeNode := htmlquery.FindOne(doc, "//time[@datetime]"); timeNode != nil {
+		for _, attr := range timeNode.Attr {
+			if attr.Key == "datetime" {
+				return attr.Val
+			}
+		}
+	}
+
+	return ""
+}
+
+// ExtractSiteName extracts the site name using multiple strategies.
+// Tries in order: OpenGraph, Schema.org, meta tags.
+func (m *MetadataExtractor) ExtractSiteName(doc *html.Node) string {
+	if doc == nil {
+		return ""
+	}
+
+	if siteName := m.getMetaContent(doc, "property", "og:site_name"); siteName != "" {
+		return siteName
+	}
+
+	if publisher := m.getSchemaOrgField(doc, "publisher"); publisher != "" {
+		return publisher
+	}
+
+	if siteName := m.getMetaContent(doc, "name", "application-name"); siteName != "" {
+		return siteName
+	}
+
+	return ""
+}
+
+// ExtractLanguage extracts the document language.
+// Tries in order: html lang attribute, OpenGraph, meta tags.
+func (m *MetadataExtractor) ExtractLanguage(doc *html.Node) string {
+	if doc == nil {
+		return ""
+	}
+
+	if htmlNode := htmlquery.FindOne(doc, "//html"); htmlNode != nil {
+		for _, attr := range htmlNode.Attr {
+			if attr.Key == "lang" {
+				return attr.Val
+			}
+		}
+	}
+
+	if locale := m.getMetaContent(doc, "property", "og:locale"); locale != "" {
+		return locale
+	}
+
+	if lang := m.getMetaContent(doc, "http-equiv", "content-language"); lang != "" {
+		return lang
+	}
+
+	return ""
+}
+
+// getMetaContent retrieves the content attribute from a meta tag.
+// Searches for meta tags with the specified attribute name and value.
+func (m *MetadataExtractor) getMetaContent(doc *html.Node, attrName, attrValue string) string {
+	if doc == nil {
+		return ""
+	}
+
+	xpath := "//meta[@" + attrName + "='" + attrValue + "']"
+	metaNode := htmlquery.FindOne(doc, xpath)
+
+	if metaNode == nil {
+		return ""
+	}
+
+	for _, attr := range metaNode.Attr {
+		if attr.Key == "content" {
+			return strings.TrimSpace(attr.Val)
+		}
+	}
+
+	return ""
+}
+
+// getSchemaOrgField extracts a field from Schema.org JSON-LD structured data.
+func (m *MetadataExtractor) getSchemaOrgField(doc *html.Node, fieldName string) string {
+	if doc == nil {
+		return ""
+	}
+
+	scripts := htmlquery.Find(doc, "//script[@type='application/ld+json']")
+
+	for _, script := range scripts {
+		if script.FirstChild == nil || script.FirstChild.Type != html.TextNode {
+			continue
+		}
+
+		var data map[string]any
+		if err := json.Unmarshal([]byte(script.FirstChild.Data), &data); err != nil {
+			continue
+		}
+
+		context, hasContext := data["@context"]
+		typeVal, hasType := data["@type"]
+
+		if !hasContext || !hasType {
+			continue
+		}
+
+		contextStr, ok := context.(string)
+		if !ok || !strings.Contains(contextStr, "schema.org") {
+			continue
+		}
+
+		typeStr, ok := typeVal.(string)
+		if !ok || (!strings.Contains(typeStr, "Article") && !strings.Contains(typeStr, "NewsArticle")) {
+			continue
+		}
+
+		if value, exists := data[fieldName]; exists {
+			return m.extractStringValue(value)
+		}
+	}
+
+	return ""
+}
+
+// extractStringValue extracts a string from various JSON value types.
+func (m *MetadataExtractor) extractStringValue(value any) string {
+	switch v := value.(type) {
+	case string:
+		return v
+	case map[string]any:
+		if name, exists := v["name"]; exists {
+			if nameStr, ok := name.(string); ok {
+				return nameStr
+			}
+		}
+	case []any:
+		if len(v) > 0 {
+			return m.extractStringValue(v[0])
+		}
+	}
+	return ""
+}

internal/articles/metadata_test.go

···
+package articles
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestMetadataExtractor(t *testing.T) {
+	t.Run("NewMetadataExtractor", func(t *testing.T) {
+		t.Run("creates extractor", func(t *testing.T) {
+			extractor := NewMetadataExtractor()
+
+			if extractor == nil {
+				t.Fatal("Expected extractor to be created, got nil")
+			}
+		})
+	})
+
+	t.Run("ExtractTitle", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from OpenGraph", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:title" content="Article Title from OpenGraph">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			title := extractor.ExtractTitle(doc)
+
+			if title != "Article Title from OpenGraph" {
+				t.Errorf("Expected OpenGraph title, got %q", title)
+			}
+		})
+
+		t.Run("extracts from title tag", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<title>Page Title from Title Tag</title>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			title := extractor.ExtractTitle(doc)
+
+			if title != "Page Title from Title Tag" {
+				t.Errorf("Expected title tag content, got %q", title)
+			}
+		})
+
+		t.Run("extracts from h1", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<h1>Heading Title</h1>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			title := extractor.ExtractTitle(doc)
+
+			if title != "Heading Title" {
+				t.Errorf("Expected h1 content, got %q", title)
+			}
+		})
+
+		t.Run("returns empty for nil document", func(t *testing.T) {
+			title := extractor.ExtractTitle(nil)
+
+			if title != "" {
+				t.Errorf("Expected empty string for nil document, got %q", title)
+			}
+		})
+
+		t.Run("prioritizes OpenGraph over title tag", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:title" content="OpenGraph Title">
+				<title>HTML Title</title>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			title := extractor.ExtractTitle(doc)
+
+			if title != "OpenGraph Title" {
+				t.Errorf("Expected OpenGraph title to have priority, got %q", title)
+			}
+		})
+	})
+
+	t.Run("ExtractAuthor", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from OpenGraph", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:author" content="John Doe">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			author := extractor.ExtractAuthor(doc)
+
+			if author != "John Doe" {
+				t.Errorf("Expected OpenGraph author, got %q", author)
+			}
+		})
+
+		t.Run("extracts from meta tag", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta name="author" content="Jane Smith">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			author := extractor.ExtractAuthor(doc)
+
+			if author != "Jane Smith" {
+				t.Errorf("Expected meta tag author, got %q", author)
+			}
+		})
+
+		t.Run("extracts from rel=author link", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<a rel="author" href="/author/bob">Bob Johnson</a>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			author := extractor.ExtractAuthor(doc)
+
+			if author != "Bob Johnson" {
+				t.Errorf("Expected rel=author link text, got %q", author)
+			}
+		})
+
+		t.Run("extracts from byline class", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<span class="author-name">Alice Brown</span>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			author := extractor.ExtractAuthor(doc)
+
+			if author != "Alice Brown" {
+				t.Errorf("Expected byline class text, got %q", author)
+			}
+		})
+
+		t.Run("returns empty for nil document", func(t *testing.T) {
+			author := extractor.ExtractAuthor(nil)
+
+			if author != "" {
+				t.Errorf("Expected empty string for nil document, got %q", author)
+			}
+		})
+	})
+
+	t.Run("ExtractPublishedDate", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from OpenGraph", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:published_time" content="2025-01-15T10:00:00Z">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			date := extractor.ExtractPublishedDate(doc)
+
+			if date != "2025-01-15T10:00:00Z" {
+				t.Errorf("Expected OpenGraph date, got %q", date)
+			}
+		})
+
+		t.Run("extracts from article:published_time", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="article:published_time" content="2025-02-20">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			date := extractor.ExtractPublishedDate(doc)
+
+			if date != "2025-02-20" {
+				t.Errorf("Expected article:published_time, got %q", date)
+			}
+		})
+
+		t.Run("extracts from time element", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<time datetime="2025-03-25T14:30:00">March 25, 2025</time>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			date := extractor.ExtractPublishedDate(doc)
+
+			if date != "2025-03-25T14:30:00" {
+				t.Errorf("Expected time element datetime, got %q", date)
+			}
+		})
+
+		t.Run("returns empty for nil document", func(t *testing.T) {
+			date := extractor.ExtractPublishedDate(nil)
+
+			if date != "" {
+				t.Errorf("Expected empty string for nil document, got %q", date)
+			}
+		})
+	})
+
+	t.Run("ExtractSiteName", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from OpenGraph", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:site_name" content="Example News">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			siteName := extractor.ExtractSiteName(doc)
+
+			if siteName != "Example News" {
+				t.Errorf("Expected OpenGraph site_name, got %q", siteName)
+			}
+		})
+
+		t.Run("extracts from application-name", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta name="application-name" content="Tech Blog">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			siteName := extractor.ExtractSiteName(doc)
+
+			if siteName != "Tech Blog" {
+				t.Errorf("Expected application-name, got %q", siteName)
+			}
+		})
+
+		t.Run("returns empty for nil document", func(t *testing.T) {
+			siteName := extractor.ExtractSiteName(nil)
+
+			if siteName != "" {
+				t.Errorf("Expected empty string for nil document, got %q", siteName)
+			}
+		})
+	})
+
+	t.Run("ExtractLanguage", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from html lang attribute", func(t *testing.T) {
+			htmlStr := `<html lang="en-US"><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			lang := extractor.ExtractLanguage(doc)
+
+			if lang != "en-US" {
+				t.Errorf("Expected html lang attribute, got %q", lang)
+			}
+		})
+
+		t.Run("extracts from OpenGraph locale", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<meta property="og:locale" content="fr-FR">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			lang := extractor.ExtractLanguage(doc)
+
+			if lang != "fr-FR" {
+				t.Errorf("Expected OpenGraph locale, got %q", lang)
+			}
+		})
+
+		t.Run("returns empty for nil document", func(t *testing.T) {
+			lang := extractor.ExtractLanguage(nil)
+
+			if lang != "" {
+				t.Errorf("Expected empty string for nil document, got %q", lang)
+			}
+		})
+	})
+
+	t.Run("getSchemaOrgField", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts from JSON-LD Article", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<script type="application/ld+json">
+				{
+					"@context": "https://schema.org",
+					"@type": "Article",
+					"headline": "Test Article",
+					"author": "Test Author",
+					"datePublished": "2025-01-15"
+				}
+				</script>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			headline := extractor.getSchemaOrgField(doc, "headline")
+			author := extractor.getSchemaOrgField(doc, "author")
+			date := extractor.getSchemaOrgField(doc, "datePublished")
+
+			if headline != "Test Article" {
+				t.Errorf("Expected headline from JSON-LD, got %q", headline)
+			}
+
+			if author != "Test Author" {
+				t.Errorf("Expected author from JSON-LD, got %q", author)
+			}
+
+			if date != "2025-01-15" {
+				t.Errorf("Expected datePublished from JSON-LD, got %q", date)
+			}
+		})
+
+		t.Run("extracts from NewsArticle type", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<script type="application/ld+json">
+				{
+					"@context": "https://schema.org",
+					"@type": "NewsArticle",
+					"headline": "Breaking News"
+				}
+				</script>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			headline := extractor.getSchemaOrgField(doc, "headline")
+
+			if headline != "Breaking News" {
+				t.Errorf("Expected headline from NewsArticle, got %q", headline)
+			}
+		})
+
+		t.Run("handles nested author object", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<script type="application/ld+json">
+				{
+					"@context": "https://schema.org",
+					"@type": "Article",
+					"author": {
+						"@type": "Person",
+						"name": "Nested Author"
+					}
+				}
+				</script>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			author := extractor.getSchemaOrgField(doc, "author")
+
+			if author != "Nested Author" {
+				t.Errorf("Expected nested author name, got %q", author)
+			}
+		})
+
+		t.Run("returns empty for invalid JSON", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<script type="application/ld+json">
+				{ invalid json }
+				</script>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.getSchemaOrgField(doc, "headline")
+
+			if result != "" {
+				t.Errorf("Expected empty for invalid JSON, got %q", result)
+			}
+		})
+
+		t.Run("returns empty for non-Article types", func(t *testing.T) {
+			htmlStr := `<html><head>
+				<script type="application/ld+json">
+				{
+					"@context": "https://schema.org",
+					"@type": "WebPage",
+					"headline": "Not an article"
+				}
+				</script>
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.getSchemaOrgField(doc, "headline")
+
+			if result != "" {
+				t.Errorf("Expected empty for WebPage type, got %q", result)
+			}
+		})
+	})
+
+	t.Run("ExtractMetadata", func(t *testing.T) {
+		extractor := NewMetadataExtractor()
+
+		t.Run("extracts all metadata fields", func(t *testing.T) {
+			htmlStr := `<html lang="en"><head>
+				<title>Full Article Title</title>
+				<meta property="og:author" content="Full Name">
+				<meta property="article:published_time" content="2025-01-20">
+				<meta property="og:site_name" content="News Site">
+			</head><body></body></html>`
+			doc := parseHTML(htmlStr)
+
+			result := extractor.ExtractMetadata(doc)
+
+			if result == nil {
+				t.Fatal("Expected result, got nil")
+			}
+
+			if !strings.Contains(result.Title, "Full Article Title") {
+				t.Errorf("Expected title to be extracted, got %q", result.Title)
+			}
+
+			if result.Author != "Full Name" {
+				t.Errorf("Expected author to be extracted, got %q", result.Author)
+			}
+
+			if result.PublishedDate != "2025-01-20" {
+				t.Errorf("Expected date to be extracted, got %q", result.PublishedDate)
+			}
+
+			if result.SiteName != "News Site" {
+				t.Errorf("Expected site name to be extracted, got %q", result.SiteName)
+			}
+
+			if result.Language != "en" {
+				t.Errorf("Expected language to be extracted, got %q", result.Language)
+			}
+		})
+
+		t.Run("returns empty result for nil document", func(t *testing.T) {
+			result := extractor.ExtractMetadata(nil)
+
+			if result == nil {
+				t.Error("Expected empty result, got nil")
+			}
+		})
+	})
+}

internal/articles/parser.go

···
 
 // ParsedContent represents the extracted content from a web page
 type ParsedContent struct {
-	Title   string
-	Author  string
-	Date    string
-	Content string
-	URL     string
+	Title            string
+	Author           string
+	Date             string
+	Content          string
+	URL              string
+	Confidence       float64 // 0-1 scale, confidence in extraction quality
+	ExtractionMethod string  // "xpath", "heuristic", "dual-validated", etc.
 }
 
 // ParsingRule represents XPath rules for extracting content from a specific domain
···
 
 // ArticleParser implements the Parser interface
 type ArticleParser struct {
-	rules  map[string]*ParsingRule
-	client *http.Client
+	rules             map[string]*ParsingRule
+	client            *http.Client
+	heuristicExtract  *HeuristicExtractor
+	metadataExtractor *MetadataExtractor
 }
 
 // NewArticleParser creates a new ArticleParser with the specified HTTP client and loaded rules
 func NewArticleParser(client *http.Client) (*ArticleParser, error) {
 	parser := &ArticleParser{
-		rules:  make(map[string]*ParsingRule),
-		client: client,
+		rules:             make(map[string]*ParsingRule),
+		client:            client,
+		heuristicExtract:  NewHeuristicExtractor(),
+		metadataExtractor: NewMetadataExtractor(),
 	}
 
 	if err := parser.loadRules(); err != nil {
···
 // AddRule adds or replaces a parsing rule for a specific domain
 func (p *ArticleParser) AddRule(domain string, rule *ParsingRule) {
 	p.rules[domain] = rule
+}
+
+// SetHTTPClient overrides the HTTP client used for fetching article content.
+func (p *ArticleParser) SetHTTPClient(client *http.Client) {
+	p.client = client
 }
 
 func (p *ArticleParser) loadRules() error {
···
 	}
 
 	domain := parsedURL.Hostname()
-
 	rule := p.findRule(domain)
-
 	req, err := http.NewRequest(http.MethodGet, s, nil)
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
···
 	return p.Parse(string(htmlBytes), domain, s)
 }
 
-// ParseHTML extracts article content from HTML string using domain-specific rules
+// ParseHTML extracts article content from HTML string using domain-specific rules with heuristic fallback.
+// Implements dual validation: compares XPath results with heuristic extraction when rules exist.
 func (p *ArticleParser) Parse(htmlContent, domain, sourceURL string) (*ParsedContent, error) {
+	doc, err := htmlquery.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse HTML: %w", err)
+	}
+
 	rule := p.findRule(domain)
 
 	if rule == nil {
-		return nil, fmt.Errorf("no parsing rule found for domain: %s", domain)
+		return p.parseWithHeuristics(doc, sourceURL)
 	}
 
-	doc, err := htmlquery.Parse(strings.NewReader(htmlContent))
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse HTML: %w", err)
+	content := &ParsedContent{
+		URL:              sourceURL,
+		ExtractionMethod: "xpath",
+		Confidence:       0.85,
 	}
 
-	content := &ParsedContent{URL: sourceURL}
-
 	if rule.Title != "" {
 		if titleNode := htmlquery.FindOne(doc, rule.Title); titleNode != nil {
 			content.Title = strings.TrimSpace(htmlquery.InnerText(titleNode))
 		}
 	}
+	if content.Title == "" {
+		content.Title = p.metadataExtractor.ExtractTitle(doc)
+	}
 
 	if rule.Author != "" {
 		if authorNode := htmlquery.FindOne(doc, rule.Author); authorNode != nil {
 			content.Author = strings.TrimSpace(htmlquery.InnerText(authorNode))
 		}
+	}
+	if content.Author == "" {
+		content.Author = p.metadataExtractor.ExtractAuthor(doc)
 	}
 
 	if rule.Date != "" {
 		if dateNode := htmlquery.FindOne(doc, rule.Date); dateNode != nil {
 			content.Date = strings.TrimSpace(htmlquery.InnerText(dateNode))
 		}
+	}
+	if content.Date == "" {
+		content.Date = p.metadataExtractor.ExtractPublishedDate(doc)
 	}
 
 	if rule.Body != "" {
 		bodyNode := htmlquery.FindOne(doc, rule.Body)
 		if bodyNode == nil {
-			return nil, fmt.Errorf("could not extract body content from HTML")
+			return p.parseWithHeuristics(doc, sourceURL)
 		}
 
 		for _, stripXPath := range rule.Strip {
···
 
 		removeDefaultNonContentNodes(bodyNode)
 
-		content.Content = normalizeWhitespace(htmlquery.InnerText(bodyNode))
+		xpathContent := normalizeWhitespace(htmlquery.InnerText(bodyNode))
+
+		heuristicResult := p.heuristicExtract.CompareWithXPath(doc, bodyNode)
+		if heuristicResult != nil {
+			content.Content = heuristicResult.Content
+			if content.Content == "" {
+				content.Content = xpathContent
+			}
+			content.Confidence = heuristicResult.Confidence
+			content.ExtractionMethod = heuristicResult.ExtractionMethod
+		} else {
+			content.Content = xpathContent
+		}
 	}
 
 	if content.Title == "" {
 		return nil, fmt.Errorf("could not extract title from HTML")
+	}
+
+	return content, nil
+}
+
+// parseWithHeuristics performs heuristic-only extraction when no XPath rule exists.
+func (p *ArticleParser) parseWithHeuristics(doc *exhtml.Node, sourceURL string) (*ParsedContent, error) {
+	result := p.heuristicExtract.ExtractWithSemanticHTML(doc)
+	if result == nil {
+		result = &ExtractionResult{
+			ExtractionMethod: "heuristic-failed",
+			Confidence:       0.0,
+		}
+	}
+
+	metadata := p.metadataExtractor.ExtractMetadata(doc)
+	if metadata != nil {
+		if result.Title == "" {
+			result.Title = metadata.Title
+		}
+		if result.Author == "" {
+			result.Author = metadata.Author
+		}
+		if result.PublishedDate == "" {
+			result.PublishedDate = metadata.PublishedDate
+		}
+	}
+
+	content := &ParsedContent{
+		Title:            result.Title,
+		Author:           result.Author,
+		Date:             result.PublishedDate,
+		Content:          result.Content,
+		URL:              sourceURL,
+		Confidence:       result.Confidence,
+		ExtractionMethod: result.ExtractionMethod,
+	}
+
+	if content.Title == "" {
+		return nil, fmt.Errorf("could not extract title from HTML using heuristics")
+	}
+
+	if content.Confidence < 0.3 {
+		return nil, fmt.Errorf("heuristic extraction confidence too low (%.2f)", content.Confidence)
 	}
 
 	return content, nil

internal/articles/parser_test.go

···
 package articles
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"net/http"
-	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
···
 
 	"github.com/stormlightlabs/noteleaf/internal/models"
 )
-
-func newServerWithHtml(h string) *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte(h))
-	}))
-}
 
 // ExampleParser_Convert demonstrates parsing a local HTML file using Wikipedia rules.
 func ExampleParser_Convert() {
···
 
 	// Output: # Christopher Lloyd
 	//
-	// **Source:** https://en.wikipedia.org/wiki/Christopher_Lloyd
+	// **Author:** Contributors to Wikimedia projects
 }
 
 func TestArticleParser(t *testing.T) {
···
 			if err == nil {
 				t.Error("Expected error for unsupported domain")
 			}
-			if !strings.Contains(err.Error(), "no parsing rule found") {
-				t.Errorf("Expected 'no parsing rule found' error, got %v", err)
+
+			if !strings.Contains(err.Error(), "confidence too low") &&
+				!strings.Contains(err.Error(), "could not extract title") {
+				t.Errorf("Expected heuristic extraction error, got %v", err)
 			}
 		})
 
···
 			if err == nil {
 				t.Error("Expected error when no title can be extracted")
 			}
+
 			if !strings.Contains(err.Error(), "could not extract title") &&
-				!strings.Contains(err.Error(), "could not extract body content") {
-				t.Errorf("Expected title or body extraction error, got %v", err)
+				!strings.Contains(err.Error(), "could not extract body content") &&
+				!strings.Contains(err.Error(), "confidence too low") {
+				t.Errorf("Expected title, body, or confidence error, got %v", err)
 			}
 		})
 
···
 				t.Error("Expected footer content to be stripped")
 			}
 		})
+
+		t.Run("uses heuristic extraction for unsupported domain with semantic HTML", func(t *testing.T) {
+			htmlContent := `<html><head>
+				<title>Heuristic Test Article</title>
+				<meta property="og:author" content="Heuristic Author">
+				<meta property="article:published_time" content="2025-01-15">
+			</head><body>
+				<article>
+					<p>This is a substantial article that should be extracted using heuristic methods.</p>
+					<p>It contains multiple paragraphs with sufficient content for the readability algorithm.</p>
+					<p>The heuristic extractor should successfully identify this as main content.</p>
+				</article>
+			</body></html>`
+
+			markdown, err := parser.Convert(htmlContent, "unsupported-domain.com", "https://unsupported-domain.com/article")
+
+			if err == nil {
+				if !strings.Contains(markdown, "substantial article") {
+					t.Error("Expected markdown to contain extracted content")
+				}
+			}
+		})
+
+		t.Run("includes confidence score in parsed content", func(t *testing.T) {
+			htmlContent := `<html>
+			<head><title>Confidence Test</title></head>
+			<body>
+				<h1 id="firstHeading">Confidence Test Article</h1>
+				<div id="bodyContent">
+					<p>Article content for confidence testing.</p>
+				</div>
+			</body>
+		</html>`
+
+			content, err := parser.Parse(htmlContent, ".wikipedia.org", "https://en.wikipedia.org/wiki/Confidence")
+			if err != nil {
+				t.Fatalf("Expected no error, got %v", err)
+			}
+
+			if content.Confidence == 0.0 {
+				t.Error("Expected non-zero confidence score")
+			}
+
+			if content.ExtractionMethod == "" {
+				t.Error("Expected extraction method to be set")
+			}
+		})
+
+		t.Run("falls back to metadata extractor when XPath fails", func(t *testing.T) {
+			htmlContent := `<html><head>
+				<title>Metadata Fallback Test</title>
+				<meta property="og:author" content="Metadata Author">
+				<meta property="article:published_time" content="2025-01-20">
+			</head><body>
+				<h1 id="firstHeading">Fallback Test</h1>
+				<div id="bodyContent">
+					<p>Content without author or date in XPath locations.</p>
+				</div>
+			</body></html>`
+
+			content, err := parser.Parse(htmlContent, ".wikipedia.org", "https://en.wikipedia.org/wiki/Metadata_Test")
+			if err != nil {
+				t.Fatalf("Expected no error, got %v", err)
+			}
+
+			if content.Author != "Metadata Author" {
+				t.Errorf("Expected metadata fallback for author, got %q", content.Author)
+			}
+
+			if content.Date != "2025-01-20" {
+				t.Errorf("Expected metadata fallback for date, got %q", content.Date)
+			}
+		})
 	})
 
 	t.Run("ParseURL", func(t *testing.T) {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			switch {
-			case strings.Contains(r.URL.Path, "404"):
-				w.WriteHeader(http.StatusNotFound)
-			case strings.Contains(r.URL.Path, "unsupported"):
-				w.WriteHeader(http.StatusOK)
-				w.Write([]byte("<html><head><title>Test</title></head><body><p>Content</p></body></html>"))
-			default:
-				w.WriteHeader(http.StatusOK)
-				w.Write([]byte(`<html>
+		parser, err := NewArticleParser(http.DefaultClient)
+		if err != nil {
+			t.Fatalf("Failed to create parser: %v", err)
+		}
+
+		localhostRule := &ParsingRule{
+			Domain: "example.com",
+			Title:  "//h1[@id='firstHeading']",
+			Body:   "//div[@id='bodyContent']",
+			Strip:  []string{"//div[@class='noprint']"},
+		}
+		parser.AddRule("example.com", localhostRule)
+
+		const (
+			validURL       = "https://example.com/wiki/test"
+			httpErrorURL   = "https://example.com/wiki/404"
+			unsupportedURL = "https://unsupported-domain.test/article"
+		)
+
+		parser.SetHTTPClient(newMockHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			switch req.URL.String() {
+			case validURL:
+				return htmlResponse(http.StatusOK, `<html>
 					<head><title>Test Article</title></head>
 					<body>
 						<h1 id="firstHeading">Test Wikipedia Article</h1>
···
 							<div class="noprint">This gets stripped</div>
 						</div>
 					</body>
-				</html>`))
+				</html>`), nil
+			case httpErrorURL:
+				return &http.Response{
+					StatusCode: http.StatusNotFound,
+					Header:     make(http.Header),
+					Body:       io.NopCloser(strings.NewReader("")),
+				}, nil
+			case unsupportedURL:
+				return htmlResponse(http.StatusOK, `<html><head><title>Unsupported</title></head><body><p>Content</p></body></html>`), nil
+			default:
+				return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
 			}
 		}))
-		defer server.Close()
-
-		parser, err := NewArticleParser(server.Client())
-		if err != nil {
-			t.Fatalf("Failed to create parser: %v", err)
-		}
-
-		localhostRule := &ParsingRule{
-			Domain: "127.0.0.1",
-			Title:  "//h1[@id='firstHeading']",
-			Body:   "//div[@id='bodyContent']",
-			Strip:  []string{"//div[@class='noprint']"},
-		}
-		parser.AddRule("127.0.0.1", localhostRule)
 
 		t.Run("fails with invalid URL", func(t *testing.T) {
 			_, err := parser.ParseURL("not-a-url")
 			if err == nil {
 				t.Error("Expected error for invalid URL")
 			}
-			if !strings.Contains(err.Error(), "unsupported protocol scheme") {
-				t.Errorf("Expected 'unsupported protocol scheme' error, got %v", err)
+			if !strings.Contains(err.Error(), "unsupported protocol scheme") &&
+				!strings.Contains(err.Error(), "failed to fetch URL") &&
+				!strings.Contains(err.Error(), "invalid URL") {
+				t.Errorf("Expected URL scheme error, got %v", err)
 			}
 		})
 
 		t.Run("fails with unsupported domain", func(t *testing.T) {
-			_, err := parser.ParseURL(server.URL + "/unsupported.com")
+			_, err := parser.ParseURL(unsupportedURL)
 			if err == nil {
 				t.Error("Expected error for unsupported domain")
 			}
 		})
 
 		t.Run("fails with HTTP error", func(t *testing.T) {
-			_, err := parser.ParseURL(server.URL + "/404/en.wikipedia.org/wiki/test")
+			_, err := parser.ParseURL(httpErrorURL)
 			if err == nil {
 				t.Error("Expected error for HTTP 404")
+			}
+		})
+
+		t.Run("successfully parses supported domain", func(t *testing.T) {
+			content, err := parser.ParseURL(validURL)
+			if err != nil {
+				t.Fatalf("Expected no error, got %v", err)
+			}
+			if content == nil {
+				t.Fatal("Expected parsed content, got nil")
+			}
+			if content.Title != "Test Wikipedia Article" {
+				t.Errorf("Expected title to be extracted, got %q", content.Title)
+			}
+			if !strings.Contains(content.Content, "This is the article content.") {
+				t.Errorf("Expected content to include article text, got %q", content.Content)
+			}
+			if strings.Contains(content.Content, "This gets stripped") {
+				t.Error("Expected strip rules to remove non-content nodes")
 			}
 		})
 
···
 	})
 
 	t.Run("fails with unsupported domain", func(t *testing.T) {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-			w.Write([]byte("<html><head><title>Test</title></head><body><p>Content</p></body></html>"))
-		}))
-		defer server.Close()
+		unsupportedURL := "https://unsupported-domain.test/article"
+		withDefaultHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == unsupportedURL {
+				return htmlResponse(http.StatusOK, "<html><body><div>Too little content</div></body></html>"), nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		})
 
-		_, err := CreateArticleFromURL(server.URL, tempDir)
+		_, err := CreateArticleFromURL(unsupportedURL, tempDir)
 		if err == nil {
 			t.Error("Expected error for unsupported domain")
 		}
-		if !strings.Contains(err.Error(), "no parsing rule found") {
-			t.Errorf("Expected 'no parsing rule found' error, got %v", err)
+		if !strings.Contains(err.Error(), "confidence too low") &&
+			!strings.Contains(err.Error(), "could not extract title") {
+			t.Errorf("Expected heuristic extraction error, got %v", err)
 		}
 	})
 
 	t.Run("fails with HTTP error", func(t *testing.T) {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusNotFound)
-		}))
-		defer server.Close()
+		errorURL := "https://example.com/missing"
+		withDefaultHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == errorURL {
+				return &http.Response{
+					StatusCode: http.StatusNotFound,
+					Header:     make(http.Header),
+					Body:       io.NopCloser(strings.NewReader("")),
+				}, nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		})
 
-		_, err := CreateArticleFromURL("https://en.wikipedia.org/wiki/NonExistentPage12345", tempDir)
+		_, err := CreateArticleFromURL(errorURL, tempDir)
 		if err == nil {
 			t.Error("Expected error for HTTP 404")
 		}
···
 	})
 
 	t.Run("fails with network error", func(t *testing.T) {
-		_, err := CreateArticleFromURL("http://localhost:99999/test", tempDir)
+		networkURL := "https://example.com/network"
+		withDefaultHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == networkURL {
+				return nil, errors.New("dial error")
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		})
+
+		_, err := CreateArticleFromURL(networkURL, tempDir)
 		if err == nil {
 			t.Error("Expected error for network failure")
 		}
···
 	})
 
 	t.Run("fails with malformed HTML", func(t *testing.T) {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-			w.Write([]byte("<html><head><title>Test</head></body>"))
-		}))
-		defer server.Close()
-
-		parser, err := NewArticleParser(server.Client())
+		parser, err := NewArticleParser(http.DefaultClient)
 		if err != nil {
 			t.Fatalf("Failed to create parser: %v", err)
 		}
 
 		localhostRule := &ParsingRule{
-			Domain: "127.0.0.1",
+			Domain: "example.com",
 			Title:  "//h1[@id='firstHeading']",
 			Body:   "//div[@id='bodyContent']",
 			Strip:  []string{"//div[@class='noprint']"},
 		}
-		parser.AddRule("127.0.0.1", localhostRule)
+		parser.AddRule("example.com", localhostRule)
 
-		_, err = parser.ParseURL(server.URL)
+		malformedURL := "https://example.com/malformed"
+		parser.SetHTTPClient(newMockHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == malformedURL {
+				return htmlResponse(http.StatusOK, "<html><head><title>Test</head></body>"), nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		}))
+
+		_, err = parser.ParseURL(malformedURL)
 		if err == nil {
 			t.Error("Expected error for malformed HTML")
 		}
+
 		if !strings.Contains(err.Error(), "failed to parse HTML") &&
 			!strings.Contains(err.Error(), "could not extract title") &&
-			!strings.Contains(err.Error(), "could not extract body content") {
+			!strings.Contains(err.Error(), "could not extract body content") &&
+			!strings.Contains(err.Error(), "confidence too low") {
 			t.Errorf("Expected HTML parsing or extraction error, got %v", err)
 		}
 	})
 
 	t.Run("fails when no title can be extracted", func(t *testing.T) {
-		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-			w.Write([]byte(`<html>
-				<head><title>Test</title></head>
-				<body>
-					<div id="bodyContent">
-						<p>Content without proper title</p>
-					</div>
-				</body>
-			</html>`))
-		}))
-		defer server.Close()
-
-		parser, err := NewArticleParser(server.Client())
+		parser, err := NewArticleParser(http.DefaultClient)
 		if err != nil {
 			t.Fatalf("Failed to create parser: %v", err)
 		}
 
 		localhostRule := &ParsingRule{
-			Domain: "127.0.0.1",
+			Domain: "example.com",
 			Title:  "//h1[@id='firstHeading']",
 			Body:   "//div[@id='bodyContent']",
 			Strip:  []string{"//div[@class='noprint']"},
 		}
-		parser.AddRule("127.0.0.1", localhostRule)
+		parser.AddRule("example.com", localhostRule)
 
-		_, err = parser.ParseURL(server.URL)
-		if err == nil {
-			t.Error("Expected error when no title can be extracted")
-		}
-		if !strings.Contains(err.Error(), "could not extract title") {
-			t.Errorf("Expected 'could not extract title' error, got %v", err)
+		noTitleURL := "https://example.com/notitle"
+		parser.SetHTTPClient(newMockHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == noTitleURL {
+				return htmlResponse(http.StatusOK, `<html>
+					<head><title>Test</title></head>
+					<body>
+						<div id="bodyContent">
+							<p>Content without proper title</p>
+						</div>
+					</body>
+				</html>`), nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		}))
+
+		result, err := parser.ParseURL(noTitleURL)
+
+		if err != nil {
+			if !strings.Contains(err.Error(), "could not extract title") &&
+				!strings.Contains(err.Error(), "confidence too low") {
+				t.Errorf("Expected title extraction error, got %v", err)
+			}
+		} else if result != nil {
+			if result.Title == "" {
+				t.Error("Expected title to be extracted via metadata fallback")
+			}
 		}
 	})
 
···
 			</body>
 		</html>`
 
-		server := newServerWithHtml(wikipediaHTML)
-		defer server.Close()
-
-		parser, err := NewArticleParser(server.Client())
+		parser, err := NewArticleParser(http.DefaultClient)
 		if err != nil {
 			t.Fatalf("Failed to create parser: %v", err)
 		}
 
 		localhostRule := &ParsingRule{
-			Domain: "127.0.0.1",
+			Domain: "example.com",
 			Title:  "//h1[@id='firstHeading']",
 			Body:   "//div[@id='bodyContent']",
 			Strip:  []string{"//div[@class='noprint']"},
 		}
-		parser.AddRule("127.0.0.1", localhostRule)
+		parser.AddRule("example.com", localhostRule)
 
-		content, err := parser.ParseURL(server.URL)
+		contentURL := "https://example.com/integration"
+		parser.SetHTTPClient(newMockHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == contentURL {
+				return htmlResponse(http.StatusOK, wikipediaHTML), nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		}))
+
+		content, err := parser.ParseURL(contentURL)
 		if err != nil {
 			t.Fatalf("Expected no error, got %v", err)
 		}
···
 		}
 
 		article := &models.Article{
-			URL:          server.URL,
+			URL:          contentURL,
 			Title:        content.Title,
 			MarkdownPath: mdPath,
 			HTMLPath:     htmlPath,
···
 		if article.Title != "Integration Test Article" {
 			t.Errorf("Expected title 'Integration Test Article', got %s", article.Title)
 		}
-		if article.URL != server.URL {
-			t.Errorf("Expected URL %s, got %s", server.URL, article.URL)
+		if article.URL != contentURL {
+			t.Errorf("Expected URL %s, got %s", contentURL, article.URL)
 		}
 		if article.MarkdownPath == "" {
 			t.Error("Expected non-empty markdown path")
···
 			</body>
 		</html>`
 
-		server := newServerWithHtml(contentHTML)
-		defer server.Close()
-
-		parser, err := NewArticleParser(server.Client())
+		parser, err := NewArticleParser(http.DefaultClient)
 		if err != nil {
 			t.Fatalf("Failed to create parser: %v", err)
 		}
 
 		localhostRule := &ParsingRule{
-			Domain: "127.0.0.1",
+			Domain: "example.com",
 			Title:  "//h1[contains(concat(' ',normalize-space(@class),' '),' title ')]",
 			Body:   "//blockquote[contains(concat(' ',normalize-space(@class),' '),' abstract ')]",
 			Date:   "//meta[@name='citation_date']/@content",
 			Author: "//meta[@name='citation_author']/@content",
 		}
-		parser.AddRule("127.0.0.1", localhostRule)
+		parser.AddRule("example.com", localhostRule)
+
+		contentURL := "https://example.com/metadata"
+		parser.SetHTTPClient(newMockHTTPClient(t, func(req *http.Request) (*http.Response, error) {
+			if req.URL.String() == contentURL {
+				return htmlResponse(http.StatusOK, contentHTML), nil
+			}
+			return nil, fmt.Errorf("unexpected request: %s", req.URL.String())
+		}))
 
-		content, err := parser.ParseURL(server.URL)
+		content, err := parser.ParseURL(contentURL)
 		if err != nil {
 			t.Fatalf("Expected no error, got %v", err)
 		}
···
 		}
 	})
 }
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func newMockHTTPClient(t *testing.T, fn roundTripFunc) *http.Client {
+	t.Helper()
+	return &http.Client{Transport: fn}
+}
+
+func htmlResponse(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode: status,
+		Header:     http.Header{"Content-Type": []string{"text/html; charset=utf-8"}},
+		Body:       io.NopCloser(strings.NewReader(body)),
+	}
+}
+
+func withDefaultHTTPClient(t *testing.T, fn roundTripFunc) {
+	t.Helper()
+	original := http.DefaultClient.Transport
+	http.DefaultClient.Transport = fn
+	t.Cleanup(func() {
+		http.DefaultClient.Transport = original
+	})
+}

internal/articles/scorer.go

···
+package articles
+
+import (
+	"math"
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// ContentScore represents the score and metadata for a content node.
+type ContentScore struct {
+	Node            *html.Node
+	Score           float64
+	TextLength      int
+	LinkDensity     float64
+	ParagraphCount  int
+	AncestorDepth   int
+	ConfidenceLevel float64
+}
+
+// Scorer implements Readability-style heuristic scoring for content extraction.
+type Scorer struct {
+	linkDensityWeight   float64
+	classWeightPositive float64
+	classWeightNegative float64
+	paragraphWeight     float64
+	ancestorDecayFactor float64
+	minContentLength    int
+	minScore            float64
+	positivePattern     *regexp.Regexp
+	negativePattern     *regexp.Regexp
+	unlikelyPattern     *regexp.Regexp
+}
+
+// NewScorer creates a new Scorer with default Readability.js-inspired weights.
+func NewScorer() *Scorer {
+	return &Scorer{
+		linkDensityWeight:   -1.0,
+		classWeightPositive: 25.0,
+		classWeightNegative: -25.0,
+		paragraphWeight:     1.0,
+		ancestorDecayFactor: 0.5,
+		minContentLength:    140,
+		minScore:            20.0,
+
+		positivePattern: regexp.MustCompile(`(?i)(article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story)`),
+		negativePattern: regexp.MustCompile(`(?i)(combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|ad-|advertisement|breadcrumb|hidden|nav|menu|header)`),
+		unlikelyPattern: regexp.MustCompile(`(?i)(banner|cookie|popup|modal)`),
+	}
+}
+
+// ScoreNode calculates a content score for the given node based on multiple heuristics.
+// This implements the core Readability scoring algorithm.
+func (s *Scorer) ScoreNode(node *html.Node) *ContentScore {
+	if node == nil || node.Type != html.ElementNode {
+		return nil
+	}
+
+	score := &ContentScore{
+		Node:          node,
+		Score:         0.0,
+		AncestorDepth: s.calculateDepth(node),
+	}
+
+	score.Score = s.getTagScore(node.Data)
+	score.Score += s.getClassIdScore(node)
+
+	score.TextLength = s.calculateTextLength(node)
+	score.LinkDensity = s.calculateLinkDensity(node)
+	score.ParagraphCount = s.countParagraphs(node)
+
+	score.Score += score.LinkDensity * s.linkDensityWeight
+	score.Score += float64(score.ParagraphCount) * s.paragraphWeight
+	score.Score += s.getTextLengthScore(score.TextLength)
+
+	score.ConfidenceLevel = s.calculateConfidence(score)
+	return score
+}
+
+// getTagScore returns a base score based on the HTML tag type.
+// Some tags are more likely to contain main content than others.
+func (s *Scorer) getTagScore(tagName string) float64 {
+	switch strings.ToLower(tagName) {
+	case "article":
+		return 30.0
+	case "section":
+		return 15.0
+	case "div":
+		return 5.0
+	case "main":
+		return 40.0
+	case "p":
+		return 3.0
+	case "pre", "td", "blockquote":
+		return 3.0
+	case "address", "ol", "ul", "dl", "dd", "dt", "li", "form":
+		return -3.0
+	case "h1", "h2", "h3", "h4", "h5", "h6", "th":
+		return -5.0
+	default:
+		return 0.0
+	}
+}
+
+// getClassIdScore analyzes class and ID attributes for positive/negative indicators.
+// Returns a positive score for content-like names, negative for navigation/ads.
+func (s *Scorer) getClassIdScore(node *html.Node) float64 {
+	score := 0.0
+	classID := s.getClassAndID(node)
+
+	if classID == "" {
+		return 0.0
+	}
+
+	if s.unlikelyPattern.MatchString(classID) {
+		return -50.0
+	}
+
+	if s.negativePattern.MatchString(classID) {
+		score += s.classWeightNegative
+	}
+
+	if s.positivePattern.MatchString(classID) {
+		score += s.classWeightPositive
+	}
+
+	return score
+}
+
+// getClassAndID concatenates class and ID attributes for pattern matching.
+func (s *Scorer) getClassAndID(node *html.Node) string {
+	var parts []string
+
+	for _, attr := range node.Attr {
+		if attr.Key == "class" || attr.Key == "id" {
+			parts = append(parts, attr.Val)
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+// calculateTextLength returns the total text length within the node.
+func (s *Scorer) calculateTextLength(node *html.Node) int {
+	text := s.getInnerText(node)
+	return len(strings.TrimSpace(text))
+}
+
+// calculateLinkDensity calculates the ratio of link text to total text.
+// Higher link density indicates navigation or related links, not main content.
+func (s *Scorer) calculateLinkDensity(node *html.Node) float64 {
+	totalText := s.getInnerText(node)
+	linkText := s.getLinkText(node)
+
+	totalLen := len(strings.TrimSpace(totalText))
+	linkLen := len(strings.TrimSpace(linkText))
+
+	if totalLen == 0 {
+		return 0.0
+	}
+
+	return float64(linkLen) / float64(totalLen)
+}
+
+// getInnerText extracts all text content from a node and its descendants.
+func (s *Scorer) getInnerText(node *html.Node) string {
+	var buf strings.Builder
+	s.extractText(node, &buf)
+	return buf.String()
+}
+
+// extractText recursively extracts text from a node tree.
+func (s *Scorer) extractText(node *html.Node, buf *strings.Builder) {
+	if node == nil {
+		return
+	}
+
+	if node.Type == html.TextNode {
+		buf.WriteString(node.Data)
+		buf.WriteString(" ")
+		return
+	}
+
+	if node.Type == html.ElementNode {
+		tag := strings.ToLower(node.Data)
+		if tag == "script" || tag == "style" || tag == "noscript" {
+			return
+		}
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		s.extractText(child, buf)
+	}
+}
+
+// getLinkText extracts text from anchor tags only.
+func (s *Scorer) getLinkText(node *html.Node) string {
+	var buf strings.Builder
+	s.extractLinkText(node, &buf)
+	return buf.String()
+}
+
+// extractLinkText recursively extracts text from anchor tags.
+func (s *Scorer) extractLinkText(node *html.Node, buf *strings.Builder) {
+	if node == nil {
+		return
+	}
+
+	if node.Type == html.ElementNode && strings.ToLower(node.Data) == "a" {
+		s.extractText(node, buf)
+		return
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		s.extractLinkText(child, buf)
+	}
+}
+
+// countParagraphs counts paragraph elements within the node.
+func (s *Scorer) countParagraphs(node *html.Node) int {
+	count := 0
+	s.walkParagraphs(node, &count)
+	return count
+}
+
+// walkParagraphs recursively counts paragraph elements.
+func (s *Scorer) walkParagraphs(node *html.Node, count *int) {
+	if node == nil {
+		return
+	}
+
+	if node.Type == html.ElementNode && strings.ToLower(node.Data) == "p" {
+		*count++
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		s.walkParagraphs(child, count)
+	}
+}
+
+// getTextLengthScore provides a bonus for nodes with substantial text content.
+func (s *Scorer) getTextLengthScore(textLen int) float64 {
+	if textLen < 25 {
+		return 0.0
+	}
+	return math.Log10(float64(textLen)) * 2.0
+}
+
+// calculateDepth calculates how deep in the DOM tree this node is.
+func (s *Scorer) calculateDepth(node *html.Node) int {
+	depth := 0
+	for n := node.Parent; n != nil; n = n.Parent {
+		depth++
+	}
+	return depth
+}
+
+// ScoreAncestors propagates scores up the DOM tree with decay.
+// This implements the Readability algorithm's ancestor scoring.
+func (s *Scorer) ScoreAncestors(scores map[*html.Node]*ContentScore, node *html.Node, baseScore float64) {
+	if node == nil || baseScore <= 0 {
+		return
+	}
+
+	currentScore := baseScore
+	level := 0
+
+	for parent := node.Parent; parent != nil && level < 5; parent = parent.Parent {
+		if parent.Type != html.ElementNode {
+			continue
+		}
+
+		if _, exists := scores[parent]; !exists {
+			scores[parent] = s.ScoreNode(parent)
+			if scores[parent] == nil {
+				continue
+			}
+		}
+
+		decayedScore := currentScore * math.Pow(s.ancestorDecayFactor, float64(level+1))
+		scores[parent].Score += decayedScore
+		level++
+	}
+}
+
+// FindTopCandidates identifies the N highest-scoring content candidates.
+func (s *Scorer) FindTopCandidates(root *html.Node, n int) []*ContentScore {
+	if root == nil || n <= 0 {
+		return nil
+	}
+
+	scores := make(map[*html.Node]*ContentScore)
+	s.scoreTree(root, scores)
+
+	var candidates []*ContentScore
+	for _, score := range scores {
+		if score.Score >= s.minScore && score.TextLength >= s.minContentLength {
+			candidates = append(candidates, score)
+		}
+	}
+
+	for i := 0; i < len(candidates); i++ {
+		for j := i + 1; j < len(candidates); j++ {
+			if candidates[j].Score > candidates[i].Score {
+				candidates[i], candidates[j] = candidates[j], candidates[i]
+			}
+		}
+	}
+
+	if len(candidates) > n {
+		candidates = candidates[:n]
+	}
+
+	return candidates
+}
+
+// scoreTree recursively scores all nodes in the tree.
+func (s *Scorer) scoreTree(node *html.Node, scores map[*html.Node]*ContentScore) {
+	if node == nil {
+		return
+	}
+
+	if node.Type == html.ElementNode {
+		tag := strings.ToLower(node.Data)
+		if tag != "script" && tag != "style" && tag != "noscript" {
+			score := s.ScoreNode(node)
+			if score != nil && score.Score > 0 {
+				scores[node] = score
+				s.ScoreAncestors(scores, node, score.Score)
+			}
+		}
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		s.scoreTree(child, scores)
+	}
+}
+
+// calculateConfidence estimates how confident we are in this content selection (between 0 & 1).
+func (s *Scorer) calculateConfidence(score *ContentScore) float64 {
+	if score == nil {
+		return 0.0
+	}
+
+	confidence := 0.0
+
+	if score.Score > s.minScore*2 {
+		confidence += 0.3
+	} else if score.Score > s.minScore {
+		confidence += 0.15
+	}
+
+	if score.TextLength > s.minContentLength*3 {
+		confidence += 0.3
+	} else if score.TextLength > s.minContentLength {
+		confidence += 0.15
+	}
+
+	if score.LinkDensity < 0.2 {
+		confidence += 0.2
+	} else if score.LinkDensity < 0.4 {
+		confidence += 0.1
+	}
+
+	if score.ParagraphCount >= 3 {
+		confidence += 0.2
+	} else if score.ParagraphCount >= 1 {
+		confidence += 0.1
+	}
+
+	if confidence > 1.0 {
+		confidence = 1.0
+	}
+
+	return confidence
+}
+
+// IsProbablyReadable determines if a document is likely to have extractable content.
+// This is inspired by Readability.js's isProbablyReaderable function.
+func (s *Scorer) IsProbablyReadable(doc *html.Node) bool {
+	if doc == nil {
+		return false
+	}
+
+	paragraphCount := s.countParagraphs(doc)
+	textLength := s.calculateTextLength(doc)
+	return paragraphCount >= 3 && textLength >= s.minContentLength
+}

internal/articles/scorer_test.go

···
+package articles
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func parseHTML(htmlStr string) *html.Node {
+	doc, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		return nil
+	}
+	return doc
+}
+
+func findElement(node *html.Node, tagName string) *html.Node {
+	if node == nil {
+		return nil
+	}
+
+	if node.Type == html.ElementNode && strings.EqualFold(node.Data, tagName) {
+		return node
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if result := findElement(child, tagName); result != nil {
+			return result
+		}
+	}
+
+	return nil
+}
+
+func findElementWithClass(node *html.Node, className string) *html.Node {
+	if node == nil {
+		return nil
+	}
+
+	if node.Type == html.ElementNode {
+		for _, attr := range node.Attr {
+			if attr.Key == "class" && strings.Contains(attr.Val, className) {
+				return node
+			}
+		}
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if result := findElementWithClass(child, className); result != nil {
+			return result
+		}
+	}
+
+	return nil
+}
+
+func TestScorer(t *testing.T) {
+	t.Run("NewScorer", func(t *testing.T) {
+		t.Run("creates scorer with default weights", func(t *testing.T) {
+			scorer := NewScorer()
+
+			if scorer == nil {
+				t.Fatal("Expected scorer to be created, got nil")
+			}
+
+			if scorer.minContentLength != 140 {
+				t.Errorf("Expected minContentLength 140, got %d", scorer.minContentLength)
+			}
+
+			if scorer.minScore != 20.0 {
+				t.Errorf("Expected minScore 20.0, got %f", scorer.minScore)
+			}
+		})
+	})
+
+	t.Run("ScoreNode", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("scores article tag highly", func(t *testing.T) {
+			htmlStr := `<html><body><article class="main-content">Article content</article></body></html>`
+			doc := parseHTML(htmlStr)
+			article := findElement(doc, "article")
+
+			score := scorer.ScoreNode(article)
+
+			if score == nil {
+				t.Fatal("Expected score, got nil")
+			}
+
+			if score.Score <= 0 {
+				t.Errorf("Expected positive score for article tag, got %f", score.Score)
+			}
+		})
+
+		t.Run("penalizes navigation elements", func(t *testing.T) {
+			htmlStr := `<html><body><div class="navigation sidebar">Nav</div></body></html>`
+			doc := parseHTML(htmlStr)
+			nav := findElementWithClass(doc, "navigation")
+
+			score := scorer.ScoreNode(nav)
+
+			if score == nil {
+				t.Fatal("Expected score, got nil")
+			}
+
+			if score.Score >= 0 {
+				t.Errorf("Expected negative score for navigation, got %f", score.Score)
+			}
+		})
+
+		t.Run("calculates text length", func(t *testing.T) {
+			htmlStr := `<html><body><div>This is some test content with multiple words</div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			score := scorer.ScoreNode(div)
+
+			if score == nil {
+				t.Fatal("Expected score, got nil")
+			}
+
+			if score.TextLength == 0 {
+				t.Error("Expected non-zero text length")
+			}
+		})
+
+		t.Run("returns nil for text nodes", func(t *testing.T) {
+			textNode := &html.Node{Type: html.TextNode, Data: "text"}
+			score := scorer.ScoreNode(textNode)
+
+			if score != nil {
+				t.Error("Expected nil score for text node")
+			}
+		})
+	})
+
+	t.Run("calculateLinkDensity", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("calculates high link density", func(t *testing.T) {
+			htmlStr := `<html><body><div><a href="#">link1</a> <a href="#">link2</a></div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			density := scorer.calculateLinkDensity(div)
+
+			if density < 0.5 {
+				t.Errorf("Expected high link density (>0.5), got %f", density)
+			}
+		})
+
+		t.Run("calculates low link density", func(t *testing.T) {
+			htmlStr := `<html><body><div>Lots of regular text content here with just <a href="#">one link</a> in it</div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			density := scorer.calculateLinkDensity(div)
+
+			if density > 0.3 {
+				t.Errorf("Expected low link density (<0.3), got %f", density)
+			}
+		})
+
+		t.Run("returns zero for empty content", func(t *testing.T) {
+			htmlStr := `<html><body><div></div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			density := scorer.calculateLinkDensity(div)
+
+			if density != 0.0 {
+				t.Errorf("Expected zero density for empty content, got %f", density)
+			}
+		})
+	})
+
+	t.Run("getClassIdScore", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("positive score for content class", func(t *testing.T) {
+			node := &html.Node{
+				Type: html.ElementNode,
+				Data: "div",
+				Attr: []html.Attribute{{Key: "class", Val: "article-content"}},
+			}
+
+			score := scorer.getClassIdScore(node)
+
+			if score <= 0 {
+				t.Errorf("Expected positive score for content class, got %f", score)
+			}
+		})
+
+		t.Run("negative score for sidebar class", func(t *testing.T) {
+			node := &html.Node{
+				Type: html.ElementNode,
+				Data: "div",
+				Attr: []html.Attribute{{Key: "class", Val: "sidebar"}},
+			}
+
+			score := scorer.getClassIdScore(node)
+
+			if score >= 0 {
+				t.Errorf("Expected negative score for sidebar class, got %f", score)
+			}
+		})
+
+		t.Run("strong negative score for banner", func(t *testing.T) {
+			node := &html.Node{
+				Type: html.ElementNode,
+				Data: "div",
+				Attr: []html.Attribute{{Key: "id", Val: "banner"}},
+			}
+
+			score := scorer.getClassIdScore(node)
+
+			if score > -30 {
+				t.Errorf("Expected strong negative score for banner, got %f", score)
+			}
+		})
+	})
+
+	t.Run("countParagraphs", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("counts multiple paragraphs", func(t *testing.T) {
+			htmlStr := `<html><body><div><p>First</p><p>Second</p><p>Third</p></div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			count := scorer.countParagraphs(div)
+
+			if count != 3 {
+				t.Errorf("Expected 3 paragraphs, got %d", count)
+			}
+		})
+
+		t.Run("returns zero for no paragraphs", func(t *testing.T) {
+			htmlStr := `<html><body><div>Just text</div></body></html>`
+			doc := parseHTML(htmlStr)
+			div := findElement(doc, "div")
+
+			count := scorer.countParagraphs(div)
+
+			if count != 0 {
+				t.Errorf("Expected 0 paragraphs, got %d", count)
+			}
+		})
+	})
+
+	t.Run("FindTopCandidates", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("finds article with substantial content", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<article class="main-content">
+					<p>This is a long paragraph with substantial content that should score well in the readability algorithm.</p>
+					<p>This is another paragraph with more content to increase the score.</p>
+					<p>And a third paragraph to ensure we have enough text and structure.</p>
+				</article>
+				<aside class="sidebar">
+					<a href="#">Link</a>
+				</aside>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			candidates := scorer.FindTopCandidates(doc, 5)
+
+			if len(candidates) == 0 {
+				t.Fatal("Expected to find candidates")
+			}
+
+			topScore := candidates[0]
+			if topScore.Score <= 0 {
+				t.Errorf("Expected positive score for top candidate, got %f", topScore.Score)
+			}
+
+			if topScore.ParagraphCount < 3 {
+				t.Errorf("Expected top candidate to contain paragraphs, got %d", topScore.ParagraphCount)
+			}
+		})
+
+		t.Run("filters out low-scoring nodes", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<div class="ad">Short ad</div>
+				<nav class="menu"><a href="#">Link</a></nav>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			candidates := scorer.FindTopCandidates(doc, 5)
+
+			for _, candidate := range candidates {
+				if candidate.Score < scorer.minScore {
+					t.Errorf("Expected all candidates to meet minimum score, got %f", candidate.Score)
+				}
+				if candidate.TextLength < scorer.minContentLength {
+					t.Errorf("Expected all candidates to meet minimum length, got %d", candidate.TextLength)
+				}
+			}
+		})
+
+		t.Run("returns empty for nil root", func(t *testing.T) {
+			candidates := scorer.FindTopCandidates(nil, 5)
+
+			if candidates != nil {
+				t.Error("Expected nil for nil root")
+			}
+		})
+	})
+
+	t.Run("calculateConfidence", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("high confidence for good content", func(t *testing.T) {
+			score := &ContentScore{
+				Score:          60.0,
+				TextLength:     500,
+				LinkDensity:    0.1,
+				ParagraphCount: 5,
+			}
+
+			confidence := scorer.calculateConfidence(score)
+
+			if confidence < 0.5 {
+				t.Errorf("Expected high confidence (>0.5) for good content, got %f", confidence)
+			}
+
+			if confidence > 1.0 {
+				t.Errorf("Expected confidence <= 1.0, got %f", confidence)
+			}
+		})
+
+		t.Run("low confidence for poor content", func(t *testing.T) {
+			score := &ContentScore{
+				Score:          10.0,
+				TextLength:     50,
+				LinkDensity:    0.8,
+				ParagraphCount: 0,
+			}
+
+			confidence := scorer.calculateConfidence(score)
+
+			if confidence > 0.3 {
+				t.Errorf("Expected low confidence (<0.3) for poor content, got %f", confidence)
+			}
+		})
+
+		t.Run("returns zero for nil score", func(t *testing.T) {
+			confidence := scorer.calculateConfidence(nil)
+
+			if confidence != 0.0 {
+				t.Errorf("Expected 0.0 for nil score, got %f", confidence)
+			}
+		})
+	})
+
+	t.Run("IsProbablyReadable", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("returns true for readable document", func(t *testing.T) {
+			htmlStr := `<html><body>
+				<article>
+					<p>First paragraph with sufficient text content to be considered readable.</p>
+					<p>Second paragraph with more text.</p>
+					<p>Third paragraph with additional content.</p>
+				</article>
+			</body></html>`
+			doc := parseHTML(htmlStr)
+
+			readable := scorer.IsProbablyReadable(doc)
+
+			if !readable {
+				t.Error("Expected document to be readable")
+			}
+		})
+
+		t.Run("returns false for short document", func(t *testing.T) {
+			htmlStr := `<html><body><div>Short</div></body></html>`
+			doc := parseHTML(htmlStr)
+
+			readable := scorer.IsProbablyReadable(doc)
+
+			if readable {
+				t.Error("Expected document to not be readable")
+			}
+		})
+
+		t.Run("returns false for nil document", func(t *testing.T) {
+			readable := scorer.IsProbablyReadable(nil)
+
+			if readable {
+				t.Error("Expected nil document to not be readable")
+			}
+		})
+	})
+
+	t.Run("ScoreAncestors", func(t *testing.T) {
+		scorer := NewScorer()
+
+		t.Run("propagates score to parent nodes", func(t *testing.T) {
+			htmlStr := `<html><body><div><article><p>Content</p></article></div></body></html>`
+			doc := parseHTML(htmlStr)
+			p := findElement(doc, "p")
+
+			scores := make(map[*html.Node]*ContentScore)
+			scores[p] = &ContentScore{Node: p, Score: 10.0}
+
+			scorer.ScoreAncestors(scores, p, 100.0)
+
+			article := findElement(doc, "article")
+			if scores[article] == nil {
+				t.Error("Expected article to receive propagated score")
+			}
+
+			if scores[article].Score <= 0 {
+				t.Errorf("Expected positive propagated score, got %f", scores[article].Score)
+			}
+		})
+	})
+}

internal/services/services_test.go

···
 	"context"
 	"encoding/json"
 	"net/http"
-	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/stormlightlabs/noteleaf/internal/shared"
 	"golang.org/x/time/rate"
 )
 
···
 				},
 			}
 
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				if r.URL.Path != "/search.json" {
 					t.Errorf("Expected path /search.json, got %s", r.URL.Path)
 				}
···
 		})
 
 		t.Run("handles API error", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusInternalServerError)
 			}))
 			defer server.Close()
···
 		})
 
 		t.Run("handles malformed JSON", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.Header().Set("Content-Type", "application/json")
 				w.Write([]byte("invalid json"))
 			}))
···
 				Covers:      []int{8739161, 8739162},
 			}
 
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				if !strings.HasPrefix(r.URL.Path, "/works/") {
 					t.Errorf("Expected path to start with /works/, got %s", r.URL.Path)
 				}
···
 		})
 
 		t.Run("handles not found", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusNotFound)
 			}))
 			defer server.Close()
···
 		})
 
 		t.Run("handles API error", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusInternalServerError)
 			}))
 			defer server.Close()
···
 
 	t.Run("Check", func(t *testing.T) {
 		t.Run("successful check", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				if r.URL.Path != "/search.json" {
 					t.Errorf("Expected path /search.json, got %s", r.URL.Path)
 				}
···
 		})
 
 		t.Run("handles API failure", func(t *testing.T) {
-			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			server := shared.NewHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				w.WriteHeader(http.StatusServiceUnavailable)
 			}))
 			defer server.Close()

internal/shared/test_utilities.go

···
 
 import (
 	"encoding/json"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"os"
···
 	"testing"
 	"time"
 )
+
+func NewHTTPTestServer(t testing.TB, handler http.Handler) *httptest.Server {
+	t.Helper()
+	server, err := startHTTPTestServer(handler)
+	if err != nil {
+		t.Fatalf("failed to start HTTP test server: %v", err)
+	}
+	return server
+}
+
+func startHTTPTestServer(handler http.Handler) (*httptest.Server, error) {
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		return nil, err
+	}
+	server := &httptest.Server{
+		Listener: ln,
+		Config:   &http.Server{Handler: handler},
+	}
+	server.Start()
+	return server, nil
+}
 
 func CreateTempDir(p string, t *testing.T) (string, func()) {
 	t.Helper()
···
 
 // WithHandler sets up the mock server with a custom handler
 func (m *HTTPMockServer) WithHandler(handler http.HandlerFunc) *HTTPMockServer {
-	m.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	server, err := startHTTPTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		m.requests = append(m.requests, r)
 		handler(w, r)
 	}))
+	if err != nil {
+		panic(err)
+	}
+
+	m.server = server
 	return m
 }