# Final Updated Code (All Files)

Below are the complete, production‑ready files with all fixes applied.  
All protocol‑specific identifiers (MCEGP‑BOUND, REQ‑XYZ, audit IDs) have been removed.  
The code is self‑contained and documented in English.

---

## `requirements_forum.txt`

```txt
# Forum Skill Dependencies

# Core scraping and parsing
forumscraper>=1.0.0,<2.0.0   # XenForo scraping library
beautifulsoup4>=4.12.0       # HTML parsing fallback
requests>=2.31.0             # HTTP client for fallback scraper
cloudscraper>=1.2.0          # Cloudflare bypass (optional but recommended)

# Analysis
nltk>=3.8.0                  # Vocabulary statistics, tokenization
scikit-learn>=1.3.0          # TF-IDF, optional LDA
# If scikit-learn is not available, a simple TF-IDF fallback is used.

# Utility
python-dateutil>=2.8.2       # Robust timestamp parsing
portalocker>=2.7.0           # File locking for checkpoint

# Testing (optional)
pytest>=7.0.0
pytest-mock>=3.0.0

"""
XenForo Forum Scraper – downloads public posts, profile, and social interactions.
"""

import argparse
import json
import os
import sys
import time
import random
import re
from dataclasses import dataclass, asdict
from datetime import datetime
from typing import List, Optional, Dict, Any, Tuple

# File locking for checkpoint
try:
    import portalocker
except ImportError:
    portalocker = None
    print("Warning: portalocker not installed. Checkpoint may be unsafe.")

# Fallback scraping
try:
    import requests
    from bs4 import BeautifulSoup
    HAS_FALLBACK = True
except ImportError:
    HAS_FALLBACK = False

# Cloudflare bypass
try:
    import cloudscraper
    HAS_CLOUDSCRAPER = True
except ImportError:
    HAS_CLOUDSCRAPER = False


@dataclass
class UserProfile:
    id: int
    username: str
    joinDate: str
    postCount: int
    reactionScore: int
    avatarUrl: Optional[str] = None
    signature: Optional[str] = None
    about: Optional[str] = None


@dataclass
class Post:
    id: int
    threadId: int
    threadTitle: str
    forumId: int
    forumName: str
    content: str
    timestamp: str
    reactions: Dict[str, int]
    isOp: bool
    parentPostId: Optional[int] = None
    mentions: List[str] = None


@dataclass
class Corpus:
    user: UserProfile
    posts: List[Post]
    threadsStarted: List[int]
    repliesReceived: List[int]
    scraped_at: str


class CloudflareException(Exception):
    """Raised when a Cloudflare challenge is detected."""
    pass


class ForumScraper:
    def __init__(self, forum_url: str, output_dir: str, max_posts: int = 5000,
                 cookie_file: Optional[str] = None, rate_limit: float = 1.0):
        self.forum_url = forum_url.rstrip('/')
        self.output_dir = output_dir
        self.max_posts = max_posts
        self.cookie_file = cookie_file
        self.rate_limit = rate_limit
        self.session = None
        self.cookies = None
        self.seen_post_ids = set()
        self.checkpoint_file = None
        self.last_page = 0
        self.stored_max_posts = max_posts

        os.makedirs(output_dir, exist_ok=True)
        self.corpus_dir = os.path.join(output_dir, 'corpus')
        os.makedirs(self.corpus_dir, exist_ok=True)
        self.checkpoint_file = os.path.join(self.corpus_dir, 'checkpoint.json')

        if cookie_file and os.path.exists(cookie_file):
            self._load_cookies(cookie_file)

        if HAS_FALLBACK:
            if HAS_CLOUDSCRAPER:
                self.session = cloudscraper.create_scraper()
            else:
                self.session = requests.Session()
            if self.cookies:
                self.session.cookies.update(self.cookies)

    def _load_cookies(self, cookie_file: str) -> None:
        try:
            import http.cookiejar as cookielib
            cj = cookielib.MozillaCookieJar()
            cj.load(cookie_file, ignore_expires=True, ignore_discard=True)
            self.cookies = {c.name: c.value for c in cj}
        except Exception as e:
            print(f"Warning: Could not load cookies from {cookie_file}: {e}")

    def _resolve_username_to_id(self, username: str) -> int:
        candidates = []
        profile_urls = [
            f"{self.forum_url}/members/{username}/",
            f"{self.forum_url}/members/{username}",
        ]
        for url in profile_urls:
            response = self._fetch_url(url)
            if response and response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                link = soup.find('a', href=re.compile(r'/members/.*\.\d+/'))
                if link:
                    href = link.get('href')
                    match = re.search(r'/members/.*\.(\d+)/', href)
                    if match:
                        candidates.append(int(match.group(1)))
                meta = soup.find('meta', {'property': 'og:url'})
                if meta:
                    url_content = meta.get('content', '')
                    match = re.search(r'/members/.*\.(\d+)/', url_content)
                    if match:
                        candidates.append(int(match.group(1)))
        if not candidates:
            search_url = f"{self.forum_url}/search/member?user_id=0&username={username}"
            sr = self._fetch_url(search_url)
            if sr and sr.status_code == 200:
                soup = BeautifulSoup(sr.text, 'html.parser')
                for link in soup.find_all('a', href=True):
                    if '/members/' in link['href']:
                        match = re.search(r'/members/.*\.(\d+)/', link['href'])
                        if match:
                            candidates.append(int(match.group(1)))
        candidates = list(set(candidates))
        if not candidates:
            sys.exit(f"ERROR: User '{username}' not found on {self.forum_url}.")
        if len(candidates) > 1:
            print(f"Found multiple users with username '{username}':")
            for idx, uid in enumerate(candidates):
                prof_url = f"{self.forum_url}/members/{uid}/"
                resp = self._fetch_url(prof_url)
                join_date = "unknown"
                if resp and resp.status_code == 200:
                    soup = BeautifulSoup(resp.text, 'html.parser')
                    for dl in soup.find_all('dl', class_='pairs pairs--justified'):
                        dt = dl.find('dt')
                        dd = dl.find('dd')
                        if dt and dd and 'Joined' in dt.text:
                            join_date = dd.text.strip()
                            break
                print(f"  {idx+1}. ID {uid} (joined {join_date})")
            while True:
                choice = input("Enter the number of the correct user: ")
                try:
                    idx = int(choice) - 1
                    if 0 <= idx < len(candidates):
                        return candidates[idx]
                except ValueError:
                    pass
                print("Invalid choice, please try again.")
        return candidates[0]

    def _fetch_url(self, url: str) -> Optional[requests.Response]:
        if not HAS_FALLBACK:
            raise RuntimeError("requests and beautifulsoup4 are required.")
        attempt = 0
        max_attempts = 5
        while attempt < max_attempts:
            try:
                time.sleep(self.rate_limit)
                resp = self.session.get(url, timeout=30)
                if 'cf-browser-verification' in resp.text or 'cf-challenge' in resp.text:
                    if HAS_CLOUDSCRAPER:
                        print("Cloudflare detected; retrying with cloudscraper...")
                        self.session = cloudscraper.create_scraper()
                        resp = self.session.get(url, timeout=30)
                        if 'cf-browser-verification' not in resp.text:
                            return resp
                    raise CloudflareException(
                        "Cloudflare challenge could not be bypassed. "
                        "Please provide cookies with --cookie-file."
                    )
                if resp.status_code == 403 and 'login' in resp.text.lower():
                    print("ERROR: Profile is private or requires login.")
                    print("Please provide cookies with --cookie-file or try manual entry.")
                    sys.exit(3)
                if resp.status_code in (429, 503):
                    wait = (2 ** attempt) + random.uniform(0, 1)
                    print(f"Rate limit (status {resp.status_code}), waiting {wait:.2f}s...")
                    time.sleep(wait)
                    attempt += 1
                    continue
                if resp.status_code >= 500:
                    wait = (2 ** attempt) + random.uniform(0, 1)
                    print(f"Server error {resp.status_code}, retrying in {wait:.2f}s...")
                    time.sleep(wait)
                    attempt += 1
                    continue
                if resp.status_code == 404:
                    return None
                if resp.status_code != 200:
                    print(f"Unexpected status {resp.status_code} for {url}")
                    return None
                return resp
            except CloudflareException:
                raise
            except Exception as e:
                print(f"Request failed: {e}")
                attempt += 1
                if attempt >= max_attempts:
                    raise
                time.sleep(2 ** attempt)
        return None

    def _scrape_with_fallback(self, user_id: int) -> Tuple[UserProfile, List[Post]]:
        if not HAS_FALLBACK:
            raise ImportError("requests and beautifulsoup4 required.")
        profile_url = f"{self.forum_url}/members/{user_id}/"
        resp = self._fetch_url(profile_url)
        if not resp or resp.status_code != 200:
            sys.exit(f"ERROR: Cannot fetch profile for user ID {user_id}.")
        soup = BeautifulSoup(resp.text, 'html.parser')

        username = soup.find('h1', class_='p-title-value')
        if username:
            username = username.text.strip()
        else:
            title = soup.find('title')
            if title:
                username = title.text.split('|')[0].strip()
        if not username:
            username = f"user_{user_id}"

        stats = {}
        for dl in soup.find_all('dl', class_='pairs pairs--justified'):
            dt = dl.find('dt')
            dd = dl.find('dd')
            if dt and dd:
                key = dt.text.strip()
                value = dd.text.strip()
                stats[key] = value
        if 'Messages' not in stats:
            msg_match = re.search(r'Messages:\s*([\d,]+)', resp.text)
            if msg_match:
                stats['Messages'] = msg_match.group(1)
        if 'Reaction score' not in stats:
            react_match = re.search(r'Reaction score:\s*([\d,]+)', resp.text)
            if react_match:
                stats['Reaction score'] = react_match.group(1)

        join_date = stats.get('Joined', '')
        post_count = int(re.sub(r'[^\d]', '', stats.get('Messages', '0'))) if stats.get('Messages') else 0
        reaction_score = int(re.sub(r'[^\d]', '', stats.get('Reaction score', '0'))) if stats.get('Reaction score') else 0

        avatar_img = soup.find('img', class_='avatar')
        avatar_url = avatar_img.get('src') if avatar_img else None

        signature_div = soup.find('div', class_='signature')
        signature = signature_div.text.strip() if signature_div else None

        about_div = soup.find('div', class_='about')
        about = about_div.text.strip() if about_div else None

        profile = UserProfile(
            id=user_id,
            username=username,
            joinDate=join_date,
            postCount=post_count,
            reactionScore=reaction_score,
            avatarUrl=avatar_url,
            signature=signature,
            about=about
        )

        # Find the correct posts feed
        feed_urls = [
            f"{self.forum_url}/members/{user_id}/post-activity",
            f"{self.forum_url}/members/{user_id}/recent-content",
            f"{self.forum_url}/search/member?user_id={user_id}"
        ]
        used_url = None
        for url_template in feed_urls:
            test_url = url_template if '?' in url_template else f"{url_template}?page=1"
            resp = self._fetch_url(test_url)
            if resp and resp.status_code == 200:
                used_url = url_template
                break
        if not used_url:
            sys.exit("ERROR: Could not find a working posts feed for this user.")
        print(f"Using posts feed: {used_url}")

        posts = []
        page = 1
        self.last_page = 1
        seen_thread_ids = set()

        while len(posts) < self.max_posts:
            if self._load_checkpoint():
                page = self.last_page + 1
                print(f"Resuming from page {page}")
            if '?' in used_url:
                page_url = f"{used_url}&page={page}" if 'page=' not in used_url else f"{used_url}&page={page}"
            else:
                page_url = f"{used_url}?page={page}"
            resp = self._fetch_url(page_url)
            if not resp or resp.status_code != 200:
                break
            soup = BeautifulSoup(resp.text, 'html.parser')

            selectors = ['div.block-row', 'article', 'li.searchResult', 'div.message']
            post_entries = []
            for sel in selectors:
                entries = soup.select(sel)
                if entries:
                    post_entries = entries
                    break
            if not post_entries:
                break

            for entry in post_entries:
                post_link = entry.find('a', href=re.compile(r'/posts/\d+'))
                if not post_link:
                    continue
                post_id_match = re.search(r'/posts/(\d+)', post_link['href'])
                if not post_id_match:
                    continue
                post_id = int(post_id_match.group(1))
                if post_id in self.seen_post_ids:
                    continue
                self.seen_post_ids.add(post_id)

                thread_link = entry.find('a', href=re.compile(r'/threads/'))
                if not thread_link:
                    continue
                thread_title = thread_link.text.strip()
                thread_id_match = re.search(r'/threads/.*\.(\d+)/', thread_link['href'])
                if not thread_id_match:
                    continue
                thread_id = int(thread_id_match.group(1))

                forum_link = entry.find('a', href=re.compile(r'/forums/'))
                forum_name = forum_link.text.strip() if forum_link else ''
                forum_id_match = re.search(r'/forums/.*\.(\d+)/', forum_link['href']) if forum_link else None
                forum_id = int(forum_id_match.group(1)) if forum_id_match else 0

                content_div = entry.find('div', class_='message-content')
                if not content_div:
                    content_div = entry.find('div', class_='message-body')
                if not content_div:
                    content_div = entry.find('blockquote')
                content = content_div.text.strip() if content_div else ''

                time_tag = entry.find('time')
                timestamp = time_tag.get('datetime') if time_tag else datetime.now().isoformat()

                reactions = {}
                reaction_spans = entry.find_all('span', class_=re.compile(r'reaction--'))
                for rspan in reaction_spans:
                    reaction_type = rspan.get('data-reaction-id')
                    if not reaction_type:
                        class_attr = rspan.get('class', [])
                        for cls in class_attr:
                            if cls.startswith('reaction--'):
                                reaction_type = cls.split('--')[1]
                                break
                    if reaction_type:
                        count_span = rspan.find_previous_sibling('span', class_='reactionScore')
                        if count_span:
                            try:
                                count = int(re.sub(r'[^\d]', '', count_span.text))
                            except:
                                count = 1
                        else:
                            count = 1
                        reactions[reaction_type] = count
                if not reactions:
                    reaction_span = entry.find('span', class_='reactionScore')
                    if reaction_span:
                        text = reaction_span.text.strip()
                        if text:
                            reactions['like'] = int(re.sub(r'[^\d]', '', text))

                is_op = False
                if thread_id not in seen_thread_ids:
                    is_op = True
                    seen_thread_ids.add(thread_id)

                parent_post_id = None
                reply_link = entry.find('a', href=re.compile(r'/posts/\d+'), class_='u-replyTo')
                if reply_link:
                    parent_match = re.search(r'/posts/(\d+)', reply_link['href'])
                    if parent_match:
                        parent_post_id = int(parent_match.group(1))

                mentions = re.findall(r'@(\w+)', content)

                post = Post(
                    id=post_id,
                    threadId=thread_id,
                    threadTitle=thread_title,
                    forumId=forum_id,
                    forumName=forum_name,
                    content=content,
                    timestamp=timestamp,
                    reactions=reactions,
                    isOp=is_op,
                    parentPostId=parent_post_id,
                    mentions=mentions
                )
                posts.append(post)
                if len(posts) >= self.max_posts:
                    break

            self._save_checkpoint(page)
            self.last_page = page
            page += 1

            next_link = soup.find('a', rel='next')
            if not next_link:
                break

        return profile, posts

    def _save_checkpoint(self, page: int):
        data = {
            'last_page': page,
            'seen_post_ids': list(self.seen_post_ids),
            'max_posts': self.max_posts,
            'forum_url': self.forum_url
        }
        if portalocker:
            with open(self.checkpoint_file, 'w') as f:
                portalocker.lock(f, portalocker.LOCK_EX)
                json.dump(data, f)
                portalocker.unlock(f)
        else:
            with open(self.checkpoint_file, 'w') as f:
                json.dump(data, f)

    def _load_checkpoint(self) -> bool:
        if os.path.exists(self.checkpoint_file):
            try:
                if portalocker:
                    with open(self.checkpoint_file, 'r') as f:
                        portalocker.lock(f, portalocker.LOCK_SH)
                        data = json.load(f)
                        portalocker.unlock(f)
                else:
                    with open(self.checkpoint_file, 'r') as f:
                        data = json.load(f)
                self.last_page = data.get('last_page', 0)
                self.seen_post_ids = set(data.get('seen_post_ids', []))
                stored_max = data.get('max_posts', self.max_posts)
                if stored_max != self.max_posts:
                    self.max_posts = min(stored_max, self.max_posts)
                stored_forum = data.get('forum_url')
                if stored_forum and stored_forum != self.forum_url:
                    print(f"Warning: checkpoint forum URL '{stored_forum}' differs from current '{self.forum_url}'.")
                return True
            except Exception as e:
                print(f"Could not load checkpoint: {e}")
                return False
        return False

    def scrape(self, user_id: Optional[int] = None, username: Optional[str] = None) -> Corpus:
        if username and not user_id:
            user_id = self._resolve_username_to_id(username)
        elif not user_id:
            sys.exit("ERROR: No user ID or username provided.")

        try:
            profile, posts = self._scrape_with_fallback(user_id)
        except CloudflareException as e:
            print(f"ERROR: {e}")
            print("Please provide a cookie file via --cookie-file and try again.")
            sys.exit(2)
        except Exception as e:
            print(f"Scraping failed: {e}")
            sys.exit(1)

        if not profile:
            sys.exit("ERROR: Failed to scrape user profile.")

        threads_started = [p.threadId for p in posts if p.isOp]

        corpus = Corpus(
            user=profile,
            posts=posts,
            threadsStarted=threads_started,
            repliesReceived=[],
            scraped_at=datetime.now().isoformat()
        )

        corpus_path = os.path.join(self.corpus_dir, 'raw_corpus.json')
        with open(corpus_path, 'w', encoding='utf-8') as f:
            json.dump(self._corpus_to_dict(corpus), f, indent=2, ensure_ascii=False)

        meta = {
            'low_confidence': len(posts) < 50,
            'banned': False,
            'post_count': len(posts),
            'scraped_at': corpus.scraped_at,
            'forum_url': self.forum_url
        }
        if profile.about and 'banned' in profile.about.lower():
            meta['banned'] = True
        if profile.signature and 'banned' in profile.signature.lower():
            meta['banned'] = True
        meta_path = os.path.join(self.corpus_dir, 'meta.json')
        with open(meta_path, 'w') as f:
            json.dump(meta, f, indent=2)

        print(f"Scraped {len(posts)} posts for user {profile.username} (ID {user_id})")
        if meta['low_confidence']:
            print("Warning: Low post count (<50) – persona may be less accurate.")
        if meta['banned']:
            print("Warning: User appears banned; data may be limited.")

        return corpus

    def _corpus_to_dict(self, corpus: Corpus) -> Dict:
        return {
            'user': asdict(corpus.user),
            'posts': [asdict(p) for p in corpus.posts],
            'threadsStarted': corpus.threadsStarted,
            'repliesReceived': corpus.repliesReceived,
            'scraped_at': corpus.scraped_at
        }


def main():
    parser = argparse.ArgumentParser(description='XenForo Forum Scraper')
    parser.add_argument('--forum', required=True, help='Forum base URL (e.g., https://www.thecoli.com)')
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument('--username', help='Target username')
    group.add_argument('--userid', type=int, help='Target user ID')
    parser.add_argument('--output', required=True, help='Output directory')
    parser.add_argument('--max-posts', type=int, default=5000, help='Maximum posts to scrape (default: 5000)')
    parser.add_argument('--cookie-file', help='Path to cookies.txt (Netscape format)')
    parser.add_argument('--rate', type=float, default=1.0, help='Rate limit in seconds per request (default: 1.0)')
    parser.add_argument('--resume', action='store_true', help='Resume from last checkpoint')

    args = parser.parse_args()

    scraper = ForumScraper(
        forum_url=args.forum,
        output_dir=args.output,
        max_posts=args.max_posts,
        cookie_file=args.cookie_file,
        rate_limit=args.rate
    )

    if args.resume:
        scraper._load_checkpoint()

    if args.username:
        corpus = scraper.scrape(username=args.username)
    else:
        corpus = scraper.scrape(user_id=args.userid)

    summary_path = os.path.join(args.output, 'summary.txt')
    with open(summary_path, 'w') as f:
        f.write(f"Scraped user: {corpus.user.username} (ID: {corpus.user.id})\n")
        f.write(f"Posts: {len(corpus.posts)}\n")
        f.write(f"Joined: {corpus.user.joinDate}\n")
        f.write(f"Post count: {corpus.user.postCount}\n")
        f.write(f"Reaction score: {corpus.user.reactionScore}\n")
        f.write(f"Avatar: {corpus.user.avatarUrl}\n")
        f.write(f"Signature: {corpus.user.signature}\n")
        f.write(f"About: {corpus.user.about}\n")


if __name__ == '__main__':
    main()

"""
Forum Post Analyzer – extracts statistical fingerprints from a corpus.
"""

import json
import os
import sys
import argparse
import re
from collections import Counter
from datetime import datetime
from statistics import mean, median, stdev
from typing import List, Dict, Any, Optional
from dateutil import parser as date_parser

try:
    import nltk
    from nltk.tokenize import word_tokenize, sent_tokenize
    HAS_NLTK = True
except ImportError:
    HAS_NLTK = False

try:
    from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
    from sklearn.decomposition import LatentDirichletAllocation
    HAS_SKLEARN = True
except ImportError:
    HAS_SKLEARN = False


class ForumAnalyzer:
    def __init__(self, corpus_path: str):
        self.corpus_path = corpus_path
        self.posts = []
        self.user = None
        self._load_corpus()

    def _load_corpus(self):
        with open(self.corpus_path, 'r', encoding='utf-8') as f:
            data = json.load(f)
        self.user = data.get('user', {})
        self.posts = data.get('posts', [])
        self.threads_started = data.get('threadsStarted', [])
        self.scraped_at = data.get('scraped_at', '')

    def analyze(self) -> Dict[str, Any]:
        default = {"writing": {}, "topic_affinity": {}, "temporal": {}, "interaction": {}}
        computed = {
            "writing": self._analyze_writing(),
            "topic_affinity": self._analyze_topics(),
            "temporal": self._analyze_temporal(),
            "interaction": self._analyze_interaction()
        }
        for key in default:
            default[key].update(computed.get(key, {}))
        return default

    def _analyze_writing(self) -> Dict:
        contents = [p.get('content', '') for p in self.posts if p.get('content')]
        if not contents:
            return {}
        full_text = ' '.join(contents)

        image_posts = [p for p in self.posts if p.get('content', '') and len(p.get('content', '')) < 10 and '[IMG]' in p.get('content', '')]
        image_post_count = len(image_posts)

        if HAS_NLTK:
            try:
                words = word_tokenize(full_text.lower())
                word_count = len(words)
                unique_words = len(set(words))
                ttr = unique_words / word_count if word_count > 0 else 0
                ttrs = []
                for content in contents:
                    w = word_tokenize(content.lower())
                    if len(w) > 0:
                        ttrs.append(len(set(w)) / len(w))
                moving_ttr = mean(ttrs) if ttrs else 0
            except Exception:
                ttr = 0
                moving_ttr = 0
        else:
            words = full_text.split()
            word_count = len(words)
            unique_words = len(set(words))
            ttr = unique_words / word_count if word_count > 0 else 0
            moving_ttr = 0

        if HAS_NLTK:
            sents = sent_tokenize(full_text)
            sent_lengths = [len(s.split()) for s in sents]
        else:
            sents = re.split(r'[.!?]', full_text)
            sent_lengths = [len(s.split()) for s in sents if s.strip()]
        avg_sent_len = mean(sent_lengths) if sent_lengths else 0
        med_sent_len = median(sent_lengths) if sent_lengths else 0
        std_sent_len = stdev(sent_lengths) if len(sent_lengths) > 1 else 0

        punct_counts = {
            'period': full_text.count('.'),
            'comma': full_text.count(','),
            'exclamation': full_text.count('!'),
            'question': full_text.count('?'),
            'ellipsis': full_text.count('...') + full_text.count('…')
        }

        all_caps_pattern = re.compile(r'\b[A-Z]{2,}\b')
        all_caps_words = all_caps_pattern.findall(full_text)
        total_words = len(full_text.split())
        caps_percent = (len(all_caps_words) / total_words * 100) if total_words > 0 else 0
        sentences = re.split(r'[.!?]', full_text)
        title_case = sum(1 for s in sentences if s.strip() and s.strip()[0].isupper())
        total_sentences = len([s for s in sentences if s.strip()])
        title_percent = (title_case / total_sentences * 100) if total_sentences > 0 else 0
        lower_sentences = sum(1 for s in sentences if s.strip() and s.strip().islower())
        lower_percent = (lower_sentences / total_sentences * 100) if total_sentences > 0 else 0

        emoji_pattern = re.compile(
            r'[\U0001F600-\U0001F64F\U0001F300-\U0001F5FF'
            r'\U0001F680-\U0001F6FF\U0001F1E0-\U0001F1FF'
            r'\U00002702-\U000027B0\U0000FE00-\U0000FE0F'
            r'\U0001F900-\U0001F9FF]', re.UNICODE
        )
        emojis = emoji_pattern.findall(full_text)
        emoji_counter = Counter(emojis)
        top_emojis = emoji_counter.most_common(10)

        top_bigrams = []
        top_trigrams = []
        if HAS_SKLEARN and len(contents) > 1:
            try:
                vectorizer = CountVectorizer(ngram_range=(2, 3), max_features=20, stop_words='english')
                X = vectorizer.fit_transform(contents)
                terms = vectorizer.get_feature_names_out()
                sums = X.sum(axis=0).A1
                sorted_terms = sorted(zip(terms, sums), key=lambda x: -x[1])
                top_bigrams = [term for term, count in sorted_terms if len(term.split()) == 2][:10]
                top_trigrams = [term for term, count in sorted_terms if len(term.split()) == 3][:10]
            except Exception:
                pass
        if not top_bigrams:
            words = full_text.split()
            bigrams = [' '.join(words[i:i+2]) for i in range(len(words)-1)]
            trigrams = [' '.join(words[i:i+3]) for i in range(len(words)-2)]
            bigram_counter = Counter(bigrams)
            trigram_counter = Counter(trigrams)
            top_bigrams = [bg for bg, c in bigram_counter.most_common(10)]
            top_trigrams = [tg for tg, c in trigram_counter.most_common(10)]

        slang_terms = ['cap', 'no cap', 'goat', 'mvp', 'lit', 'fire', 'dope', 'woke', 'sus', 'bet', 'facts']
        slang_counts = {term: full_text.lower().count(term) for term in slang_terms}

        year_stats = {}
        for post in self.posts:
            ts = post.get('timestamp', '')
            if ts:
                try:
                    dt = date_parser.parse(ts)
                    year = dt.year
                except:
                    continue
                if year not in year_stats:
                    year_stats[year] = []
                year_stats[year].append(post.get('content', ''))
        year_ttr = {}
        if HAS_NLTK:
            for year, contents in year_stats.items():
                text = ' '.join(contents)
                words = word_tokenize(text.lower())
                if len(words) > 0:
                    year_ttr[year] = len(set(words)) / len(words)
                else:
                    year_ttr[year] = 0

        return {
            "ttr": ttr,
            "moving_ttr": moving_ttr,
            "avg_sentence_length": avg_sent_len,
            "median_sentence_length": med_sent_len,
            "std_sentence_length": std_sent_len,
            "punctuation": punct_counts,
            "caps_percent": caps_percent,
            "title_case_percent": title_percent,
            "lowercase_sentences_percent": lower_percent,
            "top_emojis": top_emojis,
            "top_bigrams": top_bigrams,
            "top_trigrams": top_trigrams,
            "slang_counts": slang_counts,
            "yearly_ttr": year_ttr,
            "image_heavy_posts": image_post_count
        }

    def _analyze_topics(self) -> Dict:
        forum_counts = Counter([p.get('forumName', 'Unknown') for p in self.posts])
        total_posts = len(self.posts)
        forum_dist = {forum: count / total_posts if total_posts > 0 else 0 for forum, count in forum_counts.items()}

        topics_lda = []
        if HAS_SKLEARN and len(self.posts) > 10:
            try:
                docs = []
                for post in self.posts:
                    title = post.get('threadTitle', '')
                    content = post.get('content', '')
                    docs.append(title + " " + content)
                vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
                X = vectorizer.fit_transform(docs)
                lda = LatentDirichletAllocation(n_components=5, random_state=42)
                lda.fit(X)
                feature_names = vectorizer.get_feature_names_out()
                for topic_idx, topic in enumerate(lda.components_):
                    top_words = [feature_names[i] for i in topic.argsort()[-10:]]
                    topics_lda.append({"topic": topic_idx, "top_words": top_words})
            except Exception as e:
                print(f"LDA failed: {e}. Using TF-IDF fallback.")
                if HAS_SKLEARN:
                    tfidf = TfidfVectorizer(max_features=100, stop_words='english')
                    X = tfidf.fit_transform(docs)
                    terms = tfidf.get_feature_names_out()
                    sums = X.sum(axis=0).A1
                    sorted_terms = sorted(zip(terms, sums), key=lambda x: -x[1])[:20]
                    topics_lda = [{"topic": "common", "top_words": [t for t, c in sorted_terms[:10]]}]
        else:
            if HAS_SKLEARN and len(self.posts) > 1:
                docs = [p.get('content', '') for p in self.posts]
                tfidf = TfidfVectorizer(max_features=100, stop_words='english')
                X = tfidf.fit_transform(docs)
                terms = tfidf.get_feature_names_out()
                sums = X.sum(axis=0).A1
                sorted_terms = sorted(zip(terms, sums), key=lambda x: -x[1])[:20]
                topics_lda = [{"topic": "common", "top_words": [t for t, c in sorted_terms[:10]]}]
            else:
                topics_lda = []

        debate_topics = {
            'Kobe vs LeBron': ['kobe', 'lebron', 'goat', 'mvp'],
            'Drake': ['drake', 'rapper', 'overrated'],
            'NBA': ['nba', 'basketball', 'playoffs'],
            'Politics': ['trump', 'biden', 'democrat', 'republican'],
        }
        stances = {}
        for topic, keywords in debate_topics.items():
            counts = {}
            for post in self.posts:
                content = post.get('content', '').lower()
                for kw in keywords:
                    if kw in content:
                        counts[kw] = counts.get(kw, 0) + 1
            positive = ['good', 'great', 'best', 'awesome', 'love']
            negative = ['bad', 'worst', 'hate', 'terrible', 'overrated']
            pos_count = sum(1 for post in self.posts if any(word in post.get('content', '').lower() for word in positive))
            neg_count = sum(1 for post in self.posts if any(word in post.get('content', '').lower() for word in negative))
            stance = "neutral"
            if pos_count > neg_count * 1.5:
                stance = "positive"
            elif neg_count > pos_count * 1.5:
                stance = "negative"
            stances[topic] = {
                "keyword_counts": counts,
                "overall_stance": stance
            }

        return {
            "forum_distribution": forum_dist,
            "lda_topics": topics_lda,
            "stances": stances
        }

    def _analyze_temporal(self) -> Dict:
        timestamps = []
        for post in self.posts:
            ts = post.get('timestamp', '')
            if ts:
                try:
                    dt = date_parser.parse(ts)
                    timestamps.append(dt)
                except:
                    continue
        if not timestamps:
            return {}

        hours = [dt.hour for dt in timestamps]
        hour_counts = Counter(hours)
        hour_hist = [hour_counts.get(h, 0) for h in range(24)]

        days = [dt.weekday() for dt in timestamps]
        day_counts = Counter(days)
        day_percent = {day: day_counts.get(day, 0) / len(timestamps) for day in range(7)}

        if len(timestamps) > 1:
            first = min(timestamps)
            last = max(timestamps)
            delta = (last - first).days
            if delta == 0:
                avg_per_day = len(timestamps)
            else:
                avg_per_day = len(timestamps) / delta
            daily_counts = {}
            for dt in timestamps:
                date_key = dt.date()
                daily_counts[date_key] = daily_counts.get(date_key, 0) + 1
            daily_counts_list = list(daily_counts.values())
            daily_variance = stdev(daily_counts_list) if len(daily_counts_list) > 1 else 0
        else:
            avg_per_day = len(timestamps)
            daily_variance = 0

        return {
            "hour_histogram": hour_hist,
            "day_percentages": day_percent,
            "avg_posts_per_day": avg_per_day,
            "daily_variance": daily_variance
        }

    def _analyze_interaction(self) -> Dict:
        total_posts = len(self.posts)
        if total_posts == 0:
            return {}

        replies = [p for p in self.posts if p.get('isOp') == False]
        reply_ratio = len(replies) / total_posts if total_posts > 0 else 0
        warning = None
        if reply_ratio == 1.0 and total_posts > 0:
            warning = "reply_ratio may be inaccurate due to missing OP detection"

        mention_count = 0
        for post in self.posts:
            mention_count += len(post.get('mentions', []))
        mention_rate = mention_count / total_posts if total_posts > 0 else 0

        thread_ids = set(p.get('threadId', 0) for p in self.posts)
        depths = []
        for tid in thread_ids:
            if tid:
                thread_posts = [p for p in self.posts if p.get('threadId') == tid]
                depths.append(len(thread_posts))
        avg_depth = mean(depths) if depths else 0

        total_reactions = 0
        for post in self.posts:
            reactions = post.get('reactions', {})
            total_reactions += sum(reactions.values())
        avg_reactions_per_post = total_reactions / total_posts if total_posts > 0 else 0
        likes_received = sum(p.get('reactions', {}).get('like', 0) for p in self.posts)
        like_ratio = likes_received / total_posts if total_posts > 0 else 0

        result = {
            "reply_ratio": reply_ratio,
            "mention_rate": mention_rate,
            "avg_reply_depth": avg_depth,
            "avg_reactions_per_post": avg_reactions_per_post,
            "likes_per_post": like_ratio,
            "reactions_given": None
        }
        if warning:
            result["warning"] = warning
        return result


def main():
    parser = argparse.ArgumentParser(description='Analyze forum corpus')
    parser.add_argument('--corpus', required=True, help='Path to raw_corpus.json')
    parser.add_argument('--output', required=True, help='Output path for analysis_report.json')
    args = parser.parse_args()

    if not os.path.exists(args.corpus):
        print(f"ERROR: Corpus file not found: {args.corpus}")
        sys.exit(1)

    analyzer = ForumAnalyzer(args.corpus)
    report = analyzer.analyze()

    os.makedirs(os.path.dirname(args.output) or '.', exist_ok=True)

    with open(args.output, 'w', encoding='utf-8') as f:
        json.dump(report, f, indent=2, ensure_ascii=False)

    print(f"Analysis report written to {args.output}")


if __name__ == '__main__':
    main()