Chapter 11: News Aggregator Case Study

import requests
import json
from typing import Any, Dict, List, Tuple, Optional

def explore_api_structure(url, headers=None, max_depth=2):
    """
    Systematically explore an API response structure.
    Returns the response data for further inspection.
    """
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        data = response.json()
        
        print(f"API Response Analysis")
        print("=" * 60)
        print(f"URL: {url}")
        print(f"Response type: {type(data).__name__}")
        
        if isinstance(data, dict):
            print(f"Top-level keys: {list(data.keys())}")
            print(f"Total keys: {len(data)}")
            
            # Look for common data container patterns
            common_containers = ['results', 'data', 'items', 'articles', 
                               'hits', 'content', 'payload', 'response']
            found_containers = [key for key in common_containers if key in data]
            if found_containers:
                print(f"Possible data containers found: {found_containers}")
            
            # Show structure of first few fields
            print("\nFirst few fields (with truncated values):")
            for i, (key, value) in enumerate(list(data.items())[:5]):
                value_type = type(value).__name__
                if isinstance(value, (dict, list)):
                    size_info = f" (length: {len(value)})" if hasattr(value, '__len__') else ""
                    print(f"  {key}: {value_type}{size_info}")
                else:
                    str_value = str(value)
                    display_value = str_value[:50] + "..." if len(str_value) > 50 else str_value
                    print(f"  {key}: {display_value}")
        
        elif isinstance(data, list):
            print(f"Array with {len(data)} items")
            if data:
                first_item = data[0]
                print(f"First item type: {type(first_item).__name__}")
                if isinstance(first_item, dict):
                    print(f"First item keys: {list(first_item.keys())}")
        
        print("\nSample structure (truncated for readability):")
        print(json.dumps(truncate_for_display(data, max_depth), indent=2))
        print("\n")
        
        return data
        
    except requests.RequestException as e:
        print(f"Error fetching data: {e}")
        return None
    except json.JSONDecodeError as e:
        print(f"Error parsing JSON: {e}")
        return None

def truncate_for_display(obj, max_depth=2, current_depth=0):
    """Helper function to truncate nested data for readable display."""
    if current_depth >= max_depth:
        return "..."
    
    if isinstance(obj, dict):
        truncated = {}
        for i, (key, value) in enumerate(obj.items()):
            if i >= 5:
                truncated["..."] = f"({len(obj) - 5} more keys)"
                break
            truncated[key] = truncate_for_display(value, max_depth, current_depth + 1)
        return truncated
    
    elif isinstance(obj, list):
        truncated = []
        for i, item in enumerate(obj[:3]):
            truncated.append(truncate_for_display(item, max_depth, current_depth + 1))
        if len(obj) > 3:
            truncated.append(f"... ({len(obj) - 3} more items)")
        return truncated
    
    else:
        if isinstance(obj, str) and len(obj) > 50:
            return obj[:50] + "..."
        return obj

from typing import List

def normalize_newsapi(response: dict) -> List[Article]:
    """
    Transform NewsAPI response into canonical Article format.
    
    NewsAPI structure:
    - Articles in 'articles' array at root
    - Source nested as object with 'name' field
    - Timestamp in 'publishedAt' (ISO 8601)
    - Optional: author, description, urlToImage, content
    """
    # Step 1: Extract items using Chapter 10 utility
    items, meta = extract_items_and_meta(response)
    
    articles = []
    
    # Step 2: Loop through items with defensive error handling
    for item in items:
        try:
            # Step 3: Extract and validate required fields
            title = item.get("title", "").strip()
            url = item.get("url", "").strip()
            published_at = item.get("publishedAt", "")
            
            # Skip items missing required fields
            if not title or not url or not published_at:
                continue
            
            # Step 4: Extract source name from nested object
            source_name = safe_get(item, "source.name", "Unknown Source")
            
            # Step 5: Extract optional fields with safe defaults
            description = item.get("description")
            if description:
                description = description.strip()
            
            author = item.get("author")
            if author:
                author = author.strip()
            
            image_url = item.get("urlToImage")
            
            # Content preview (truncate if too long)
            content = item.get("content")
            content_preview = None
            if content:
                content_preview = content[:200] + "..." if len(content) > 200 else content
            
            # Step 6: Create canonical article
            article = Article(
                title=title,
                url=url,
                published_at=published_at,
                source_name=source_name,
                description=description,
                author=author,
                image_url=image_url,
                content_preview=content_preview
            )
            
            articles.append(article)
            
        except (ValueError, KeyError, TypeError) as e:
            # Log and skip malformed items - don't crash entire batch
            print(f"Warning: Skipping malformed NewsAPI item: {e}")
            continue
    
    return articles

import os
import requests
from typing import List, Tuple, Dict

class NewsAggregator:
    """
    Aggregates news from multiple sources with graceful failure handling.
    Each source can fail independently without affecting others.
    """
    
    def __init__(self):
        """Initialize with API credentials from environment."""
        self.newsapi_key = os.environ.get("NEWSAPI_KEY")
        self.guardian_key = os.environ.get("GUARDIAN_KEY")
    
    def fetch_newsapi(self, query: str, max_results: int = 10) -> dict:
        """Fetch articles from NewsAPI."""
        if not self.newsapi_key:
            raise ValueError("NewsAPI key not configured")
        
        url = "https://newsapi.org/v2/everything"
        params = {
            "q": query,
            "pageSize": max_results,
            "apiKey": self.newsapi_key,
            "language": "en",
            "sortBy": "publishedAt"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def fetch_guardian(self, query: str, max_results: int = 10) -> dict:
        """Fetch articles from Guardian API."""
        if not self.guardian_key:
            raise ValueError("Guardian API key not configured")
        
        url = "https://content.guardianapis.com/search"
        params = {
            "q": query,
            "page-size": max_results,
            "show-fields": "all",
            "api-key": self.guardian_key,
            "order-by": "newest"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def fetch_hackernews(self, query: str, max_results: int = 10) -> dict:
        """Fetch articles from HackerNews via Algolia API."""
        url = "https://hn.algolia.com/api/v1/search"
        params = {
            "query": query,
            "hitsPerPage": max_results,
            "tags": "story"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def search(self, query: str, max_per_source: int = 10) -> Tuple[List[Article], Dict]:
        """
        Search all available sources and return combined results.
        
        Returns:
            (articles, stats) where stats contains success/failure info
        """
        all_articles = []
        stats = {
            "query": query,
            "sources_attempted": [],
            "sources_succeeded": [],
            "sources_failed": [],
            "total_articles": 0
        }
        
        # Define sources with their fetch and normalize functions
        sources = [
            ("NewsAPI", self.fetch_newsapi, normalize_newsapi),
            ("Guardian", self.fetch_guardian, normalize_guardian),
            ("HackerNews", self.fetch_hackernews, normalize_hackernews)
        ]
        
        # Try each source independently
        for source_name, fetch_fn, normalize_fn in sources:
            stats["sources_attempted"].append(source_name)
            
            try:
                # Fetch and normalize
                response = fetch_fn(query, max_per_source)
                articles = normalize_fn(response)
                
                # Record success
                all_articles.extend(articles)
                stats["sources_succeeded"].append(source_name)
                print(f"✓ {source_name}: {len(articles)} articles")
                
            except ValueError as e:
                # Configuration error (missing API key)
                stats["sources_failed"].append(f"{source_name} (not configured)")
                print(f"⊘ {source_name}: {e}")
                
            except requests.RequestException as e:
                # Network or API error
                stats["sources_failed"].append(f"{source_name} (network error)")
                print(f"✗ {source_name}: {e}")
                
            except Exception as e:
                # Unexpected error
                stats["sources_failed"].append(f"{source_name} (error: {type(e).__name__})")
                print(f"✗ {source_name}: Unexpected error - {e}")
        
        stats["total_articles"] = len(all_articles)
        return all_articles, stats

# Create aggregator and search
aggregator = NewsAggregator()

print("=== NEWS AGGREGATOR TEST ===\n")
print("Searching for 'python programming'...\n")

articles, stats = aggregator.search("python programming", max_per_source=5)

# Display statistics
print(f"\n--- Search Statistics ---")
print(f"Sources attempted: {len(stats['sources_attempted'])}")
print(f"Sources succeeded: {len(stats['sources_succeeded'])}")
print(f"Sources failed: {len(stats['sources_failed'])}")
if stats['sources_failed']:
    print(f"Failed sources: {', '.join(stats['sources_failed'])}")
print(f"Total articles: {stats['total_articles']}")

# Display sample results
print(f"\n--- Sample Results ---")
for i, article in enumerate(articles[:3], 1):
    print(f"\n{i}. {article.title}")
    print(f"   📰 {article.source_name} • {article.format_timestamp()}")
    if article.author:
        print(f"   ✍️  {article.author}")
    print(f"   🔗 {article.url}")

=== NEWS AGGREGATOR TEST ===

Searching for 'python programming'...

✓ NewsAPI: 5 articles
✓ Guardian: 5 articles
✓ HackerNews: 5 articles

--- Search Statistics ---
Sources attempted: 3
Sources succeeded: 3
Sources failed: 0
Total articles: 15

--- Sample Results ---

1. Python 3.13 Performance Improvements
   📰 TechCrunch • January 15, 2025 at 02:30 PM
   ✍️  Sarah Chen
   🔗 https://techcrunch.com/2025/01/15/python-3-13-performance

2. Python programming language gains popularity in schools
   📰 Technology • January 14, 2025 at 03:15 PM
   ✍️  Alex Hern
   🔗 https://www.theguardian.com/technology/2025/jan/14/python-schools

3. Python 3.13 is out
   📰 HackerNews • January 15, 2025 at 08:45 AM
   ✍️  throwaway2025
   🔗 https://www.python.org/downloads/release/python-3130/

"""
News Aggregator - Multi-API Integration Case Study
Demonstrates advanced JSON processing patterns from Chapter 10.

Requirements:
- Python 3.10+
- requests library: pip install requests
- Environment variables: NEWSAPI_KEY, GUARDIAN_KEY

Usage:
    python news_aggregator.py
"""

import os
import requests
import json
from dataclasses import dataclass
from typing import Any, Dict, List, Tuple, Optional
from datetime import datetime, timezone
from itertools import groupby

# ============================================================================
# SECTION 1: UTILITY FUNCTIONS (Chapter 10)
# ============================================================================

def explore_api_structure(url, headers=None, max_depth=2):
    """Systematically explore an API response structure."""
    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        data = response.json()
        
        print(f"API Response Analysis")
        print("=" * 60)
        print(f"URL: {url}")
        print(f"Response type: {type(data).__name__}")
        
        if isinstance(data, dict):
            print(f"Top-level keys: {list(data.keys())}")
            print(f"Total keys: {len(data)}")
            
            common_containers = ['results', 'data', 'items', 'articles', 
                               'hits', 'content', 'payload', 'response']
            found_containers = [key for key in common_containers if key in data]
            if found_containers:
                print(f"Possible data containers found: {found_containers}")
            
            print("\nFirst few fields (with truncated values):")
            for i, (key, value) in enumerate(list(data.items())[:5]):
                value_type = type(value).__name__
                if isinstance(value, (dict, list)):
                    size_info = f" (length: {len(value)})" if hasattr(value, '__len__') else ""
                    print(f"  {key}: {value_type}{size_info}")
                else:
                    str_value = str(value)
                    display_value = str_value[:50] + "..." if len(str_value) > 50 else str_value
                    print(f"  {key}: {display_value}")
        
        elif isinstance(data, list):
            print(f"Array with {len(data)} items")
            if data:
                first_item = data[0]
                print(f"First item type: {type(first_item).__name__}")
                if isinstance(first_item, dict):
                    print(f"First item keys: {list(first_item.keys())}")
        
        print("\nSample structure (truncated for readability):")
        print(json.dumps(truncate_for_display(data, max_depth), indent=2))
        print("\n")
        
        return data
        
    except requests.RequestException as e:
        print(f"Error fetching data: {e}")
        return None
    except json.JSONDecodeError as e:
        print(f"Error parsing JSON: {e}")
        return None


def truncate_for_display(obj, max_depth=2, current_depth=0):
    """Helper function to truncate nested data for readable display."""
    if current_depth >= max_depth:
        return "..."
    
    if isinstance(obj, dict):
        truncated = {}
        for i, (key, value) in enumerate(obj.items()):
            if i >= 5:
                truncated["..."] = f"({len(obj) - 5} more keys)"
                break
            truncated[key] = truncate_for_display(value, max_depth, current_depth + 1)
        return truncated
    
    elif isinstance(obj, list):
        truncated = []
        for i, item in enumerate(obj[:3]):
            truncated.append(truncate_for_display(item, max_depth, current_depth + 1))
        if len(obj) > 3:
            truncated.append(f"... ({len(obj) - 3} more items)")
        return truncated
    
    else:
        if isinstance(obj, str) and len(obj) > 50:
            return obj[:50] + "..."
        return obj


COMMON_COLLECTION_KEYS = ["items", "results", "data", "articles", 
                          "hits", "content", "entries", "records"]


def extract_items_and_meta(
    api_response: Any,
    container_hints: Optional[List[str]] = None
) -> Tuple[List[Any], Dict[str, Any]]:
    """Return (items, metadata) and normalize pagination signals."""
    meta: Dict[str, Any] = {}

    if isinstance(api_response, list):
        return api_response, meta

    if not isinstance(api_response, dict):
        return [], meta

    keys = (container_hints or []) + COMMON_COLLECTION_KEYS
    container_key = None
    for key in keys:
        if key in api_response and isinstance(api_response[key], list):
            container_key = key
            break

    if container_key:
        items = api_response[container_key]
        meta = {k: v for k, v in api_response.items() if k != container_key}
    else:
        items = [api_response]
        meta = {}

    meta_obj = meta.get("meta") if isinstance(meta.get("meta"), dict) else {}
    
    next_token = (
        meta_obj.get("cursor") or
        meta.get("cursor") or
        meta.get("next") or
        None
    )
    
    total = (
        meta.get("total") or 
        meta.get("totalResults") or 
        meta.get("total_count") or 
        meta_obj.get("total") or 
        None
    )
    
    meta_norm = {"next_token": next_token, "total": total}
    return items, {**meta, **meta_norm}


def safe_get(obj: Any, path: str, default=None):
    """Dot-path lookup with safe defaults."""
    cur = obj
    for part in path.split("."):
        if not isinstance(cur, dict) or part not in cur:
            return default
        cur = cur[part]
    return cur


# ============================================================================
# SECTION 2: CANONICAL MODEL
# ============================================================================

@dataclass
class Article:
    """Canonical article format for all news sources."""
    title: str
    url: str
    published_at: str
    source_name: str
    description: Optional[str] = None
    author: Optional[str] = None
    image_url: Optional[str] = None
    content_preview: Optional[str] = None
    
    def __post_init__(self):
        """Validate required fields."""
        if not self.title or not self.title.strip():
            raise ValueError("Article title cannot be empty")
        if not self.url or not self.url.strip():
            raise ValueError("Article URL cannot be empty")
        if not self.source_name or not self.source_name.strip():
            raise ValueError("Article source_name cannot be empty")
    
    def format_timestamp(self) -> str:
        """Convert ISO timestamp to human-readable format."""
        try:
            dt = datetime.fromisoformat(self.published_at.replace('Z', '+00:00'))
            return dt.strftime("%B %d, %Y at %I:%M %p")
        except (ValueError, AttributeError):
            return self.published_at
    
    def get_domain(self) -> str:
        """Extract domain from URL."""
        try:
            from urllib.parse import urlparse
            parsed = urlparse(self.url)
            return parsed.netloc or "unknown"
        except Exception:
            return "unknown"


# ============================================================================
# SECTION 3: NORMALIZERS
# ============================================================================

def normalize_newsapi(response: dict) -> List[Article]:
    """Transform NewsAPI response into canonical format."""
    items, meta = extract_items_and_meta(response)
    articles = []
    
    for item in items:
        try:
            title = item.get("title", "").strip()
            url = item.get("url", "").strip()
            published_at = item.get("publishedAt", "")
            
            if not title or not url or not published_at:
                continue
            
            source_name = safe_get(item, "source.name", "Unknown Source")
            
            description = item.get("description")
            if description:
                description = description.strip()
            
            author = item.get("author")
            if author:
                author = author.strip()
            
            image_url = item.get("urlToImage")
            
            content = item.get("content")
            content_preview = None
            if content:
                content_preview = content[:200] + "..." if len(content) > 200 else content
            
            article = Article(
                title=title,
                url=url,
                published_at=published_at,
                source_name=source_name,
                description=description,
                author=author,
                image_url=image_url,
                content_preview=content_preview
            )
            
            articles.append(article)
            
        except (ValueError, KeyError, TypeError) as e:
            print(f"Warning: Skipping malformed NewsAPI item: {e}")
            continue
    
    return articles


def normalize_guardian(response: dict) -> List[Article]:
    """Transform Guardian API response into canonical format."""
    if not isinstance(response, dict) or "response" not in response:
        return []
    
    response_obj = response["response"]
    items, meta = extract_items_and_meta(response_obj, container_hints=["results"])
    articles = []
    
    for item in items:
        try:
            title = item.get("webTitle", "").strip()
            url = item.get("webUrl", "").strip()
            published_at = item.get("webPublicationDate", "")
            
            if not title or not url or not published_at:
                continue
            
            source_name = item.get("sectionName", "The Guardian")
            
            fields = item.get("fields", {})
            if not isinstance(fields, dict):
                fields = {}
            
            description = fields.get("trailText")
            if description:
                description = description.strip()
            
            author = fields.get("byline")
            if author:
                author = author.strip()
            
            image_url = fields.get("thumbnail")
            
            body = fields.get("bodyText")
            content_preview = None
            if body:
                content_preview = body[:200] + "..." if len(body) > 200 else body
            
            article = Article(
                title=title,
                url=url,
                published_at=published_at,
                source_name=source_name,
                description=description,
                author=author,
                image_url=image_url,
                content_preview=content_preview
            )
            
            articles.append(article)
            
        except (ValueError, KeyError, TypeError) as e:
            print(f"Warning: Skipping malformed Guardian item: {e}")
            continue
    
    return articles


def normalize_hackernews(response: dict) -> List[Article]:
    """Transform HackerNews API response into canonical format."""
    items, meta = extract_items_and_meta(response, container_hints=["hits"])
    articles = []
    
    for item in items:
        try:
            title = item.get("title", "").strip()
            url = item.get("url", "").strip()
            
            if not title:
                continue
            
            if not url:
                object_id = item.get("objectID", "")
                if object_id:
                    url = f"https://news.ycombinator.com/item?id={object_id}"
                else:
                    continue
            
            created_at_unix = item.get("created_at_i")
            if created_at_unix:
                try:
                    dt = datetime.fromtimestamp(created_at_unix, tz=timezone.utc)
                    published_at = dt.isoformat().replace("+00:00", "Z")
                except (ValueError, OSError):
                    published_at = item.get("created_at", "")
            else:
                published_at = item.get("created_at", "")
            
            if not published_at:
                continue
            
            source_name = "HackerNews"
            
            description = item.get("story_text")
            if description:
                description = description.strip()
            
            author = item.get("author")
            if author:
                author = author.strip()
            
            image_url = None
            
            content_preview = None
            if description:
                content_preview = description[:200] + "..." if len(description) > 200 else description
            
            article = Article(
                title=title,
                url=url,
                published_at=published_at,
                source_name=source_name,
                description=description,
                author=author,
                image_url=image_url,
                content_preview=content_preview
            )
            
            articles.append(article)
            
        except (ValueError, KeyError, TypeError) as e:
            print(f"Warning: Skipping malformed HackerNews item: {e}")
            continue
    
    return articles


# ============================================================================
# SECTION 4: AGGREGATOR WITH DEDUPLICATION
# ============================================================================

def deduplicate_articles(articles: List[Article]) -> List[Article]:
    """
    Remove duplicate articles using URL as unique identifier.
    
    When multiple sources report the same story, they often link to
    the same original URL. Keep the first instance encountered.
    
    For production systems, consider enhancements:
    - URL normalization (strip query parameters and fragments)
    - Fuzzy title matching for different URLs covering same story
    - Domain canonicalization (www vs non-www, http vs https)
    """
    if not articles:
        return []
    
    seen_urls = set()
    unique_articles = []
    
    for article in articles:
        if article.url not in seen_urls:
            seen_urls.add(article.url)
            unique_articles.append(article)
    
    return unique_articles


def sort_articles(
    articles: List[Article],
    sort_by: str = "recency"
) -> List[Article]:
    """Sort articles by various criteria."""
    if not articles:
        return []
    
    if sort_by == "recency":
        return sorted(articles, key=lambda a: a.published_at, reverse=True)
    elif sort_by == "source":
        return sorted(articles, key=lambda a: a.source_name)
    elif sort_by == "title":
        return sorted(articles, key=lambda a: a.title.lower())
    else:
        return articles


class NewsAggregator:
    """Aggregates news from multiple sources with graceful failure handling."""
    
    def __init__(self):
        self.newsapi_key = os.environ.get("NEWSAPI_KEY")
        self.guardian_key = os.environ.get("GUARDIAN_KEY")
    
    def fetch_newsapi(self, query: str, max_results: int = 10) -> dict:
        if not self.newsapi_key:
            raise ValueError("NewsAPI key not configured")
        
        url = "https://newsapi.org/v2/everything"
        params = {
            "q": query,
            "pageSize": max_results,
            "apiKey": self.newsapi_key,
            "language": "en",
            "sortBy": "publishedAt"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def fetch_guardian(self, query: str, max_results: int = 10) -> dict:
        if not self.guardian_key:
            raise ValueError("Guardian API key not configured")
        
        url = "https://content.guardianapis.com/search"
        params = {
            "q": query,
            "page-size": max_results,
            "show-fields": "all",
            "api-key": self.guardian_key,
            "order-by": "newest"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def fetch_hackernews(self, query: str, max_results: int = 10) -> dict:
        url = "https://hn.algolia.com/api/v1/search"
        params = {
            "query": query,
            "hitsPerPage": max_results,
            "tags": "story"
        }
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()
        return response.json()
    
    def search(
        self,
        query: str,
        max_per_source: int = 10,
        deduplicate: bool = True,
        sort_by: str = "recency"
    ) -> Tuple[List[Article], Dict]:
        """Search all sources and return combined results."""
        all_articles = []
        stats = {
            "query": query,
            "sources_attempted": [],
            "sources_succeeded": [],
            "sources_failed": [],
            "total_articles_raw": 0,
            "total_articles_deduplicated": 0,
            "duplicates_removed": 0
        }
        
        sources = [
            ("NewsAPI", self.fetch_newsapi, normalize_newsapi),
            ("Guardian", self.fetch_guardian, normalize_guardian),
            ("HackerNews", self.fetch_hackernews, normalize_hackernews)
        ]
        
        for source_name, fetch_fn, normalize_fn in sources:
            stats["sources_attempted"].append(source_name)
            
            try:
                response = fetch_fn(query, max_per_source)
                articles = normalize_fn(response)
                all_articles.extend(articles)
                stats["sources_succeeded"].append(source_name)
                print(f"✓ {source_name}: {len(articles)} articles")
                
            except ValueError as e:
                stats["sources_failed"].append(f"{source_name} (not configured)")
                print(f"⊘ {source_name}: {e}")
                
            except requests.RequestException as e:
                stats["sources_failed"].append(f"{source_name} (network error)")
                print(f"✗ {source_name}: {e}")
                
            except Exception as e:
                stats["sources_failed"].append(f"{source_name} (error: {type(e).__name__})")
                print(f"✗ {source_name}: Unexpected error - {e}")
        
        stats["total_articles_raw"] = len(all_articles)
        
        if deduplicate:
            all_articles = deduplicate_articles(all_articles)
            stats["total_articles_deduplicated"] = len(all_articles)
            stats["duplicates_removed"] = stats["total_articles_raw"] - stats["total_articles_deduplicated"]
        
        all_articles = sort_articles(all_articles, sort_by)
        
        return all_articles, stats


# ============================================================================
# SECTION 5: DISPLAY FUNCTIONS
# ============================================================================

def display_article(article: Article, index: int = None, show_description: bool = True):
    """Display a single article with consistent formatting."""
    if index is not None:
        print(f"\n{index}. {article.title}")
    else:
        print(f"\n{article.title}")
    
    print(f"   📰 {article.source_name} • {article.format_timestamp()}")
    
    if article.author:
        print(f"   ✍️  {article.author}")
    
    if show_description:
        if article.description:
            desc = article.description
            if len(desc) > 150:
                desc = desc[:150] + "..."
            print(f"   {desc}")
        elif article.content_preview:
            preview = article.content_preview
            if len(preview) > 150:
                preview = preview[:150] + "..."
            print(f"   {preview}")
    
    domain = article.get_domain()
    print(f"   🔗 {article.url}")
    print(f"   ({domain})")


def display_article_list(
    articles: List[Article],
    stats: dict = None,
    max_display: int = 10,
    group_by_source: bool = False,
    show_descriptions: bool = True
):
    """Display a list of articles with formatting and statistics."""
    if not articles:
        print("\nNo articles found.")
        return
    
    print("\n" + "=" * 70)
    print("📰 NEWS AGGREGATOR RESULTS")
    print("=" * 70)
    
    if stats:
        print(f"\nQuery: '{stats.get('query', 'N/A')}'")
        print(f"Sources: {', '.join(stats.get('sources_succeeded', []))}")
        
        if stats.get('sources_failed'):
            print(f"Failed: {', '.join(stats.get('sources_failed', []))}")
        
        print(f"Articles: {len(articles)}", end="")
        
        if stats.get('duplicates_removed', 0) > 0:
            print(f" ({stats['duplicates_removed']} duplicates removed)", end="")
        
        print()
    
    if group_by_source:
        articles_sorted = sorted(articles, key=lambda a: a.source_name)
        
        for source_name, group in groupby(articles_sorted, key=lambda a: a.source_name):
            group_list = list(group)
            print(f"\n--- {source_name} ({len(group_list)} articles) ---")
            
            for i, article in enumerate(group_list[:max_display], 1):
                display_article(article, index=i, show_description=show_descriptions)
    else:
        display_count = min(len(articles), max_display)
        
        for i, article in enumerate(articles[:display_count], 1):
            display_article(article, index=i, show_description=show_descriptions)
        
        if len(articles) > max_display:
            remaining = len(articles) - max_display
            print(f"\n... and {remaining} more articles")
    
    print("\n" + "=" * 70)


# ============================================================================
# SECTION 6: INTERACTIVE CLI
# ============================================================================

class NewsAggregatorCLI:
    """Interactive command-line interface for news aggregator."""
    
    def __init__(self):
        self.aggregator = NewsAggregator()
        self.last_results = []
        self.last_stats = {}
    
    def display_banner(self):
        """Display application banner."""
        print("\n" + "=" * 70)
        print("📰 NEWS AGGREGATOR")
        print("=" * 70)
        print("\nSearch news from NewsAPI, The Guardian, and HackerNews")
        print("\nCommands:")
        print("  search   - Search for news articles")
        print("  more            - Show more results from last search")
        print("  group           - Show last results grouped by source")
        print("  help            - Show this help message")
        print("  quit            - Exit application")
        print("=" * 70 + "\n")
    
    def search(self, query: str, max_per_source: int = 10):
        """Execute search and display results."""
        print(f"\nSearching for '{query}'...\n")
        
        articles, stats = self.aggregator.search(
            query,
            max_per_source=max_per_source,
            deduplicate=True,
            sort_by="recency"
        )
        
        self.last_results = articles
        self.last_stats = stats
        
        display_article_list(
            articles,
            stats,
            max_display=10,
            show_descriptions=True
        )
    
    def show_more(self):
        """Show more results from last search."""
        if not self.last_results:
            print("\nNo previous search results. Use 'search ' first.")
            return
        
        display_article_list(
            self.last_results,
            self.last_stats,
            max_display=len(self.last_results),
            show_descriptions=False
        )
    
    def show_grouped(self):
        """Show last results grouped by source."""
        if not self.last_results:
            print("\nNo previous search results. Use 'search ' first.")
            return
        
        display_article_list(
            self.last_results,
            self.last_stats,
            max_display=50,
            group_by_source=True,
            show_descriptions=False
        )
    
    def run(self):
        """Run interactive session."""
        self.display_banner()
        
        while True:
            try:
                user_input = input("aggregator> ").strip()
                
                if not user_input:
                    continue
                
                parts = user_input.split(maxsplit=1)
                command = parts[0].lower()
                args = parts[1] if len(parts) > 1 else ""
                
                if command in ['quit', 'exit', 'q']:
                    print("\nGoodbye!")
                    break
                
                elif command == 'help':
                    self.display_banner()
                
                elif command == 'search':
                    if not args:
                        print("Usage: search ")
                    else:
                        self.search(args)
                
                elif command == 'more':
                    self.show_more()
                
                elif command == 'group':
                    self.show_grouped()
                
                else:
                    print(f"Unknown command: {command}")
                    print("Type 'help' for available commands")
            
            except KeyboardInterrupt:
                print("\n\nGoodbye!")
                break
            
            except Exception as e:
                print(f"\nError: {e}")
                print("Type 'help' for available commands")


# ============================================================================
# MAIN ENTRY POINT
# ============================================================================

def main():
    """Entry point for news aggregator CLI."""
    cli = NewsAggregatorCLI()
    cli.run()


if __name__ == "__main__":
    main()