Large Document Reader

def split_document(text, max_tokens=8000, overlap_tokens=200):
    """Split document respecting structural boundaries."""
    # Step 1: Detect document structure
    chapters = detect_chapters(text)

    if not chapters:
        # Fallback: split by sections
        chapters = detect_sections(text)

    if not chapters:
        # Fallback: split by paragraphs with size limit
        chapters = split_by_paragraphs(text, max_tokens)

    # Step 2: Merge small adjacent sections
    merged = merge_small_sections(chapters, min_tokens=500)

    # Step 3: Split oversized sections
    final = []
    for chapter in merged:
        if count_tokens(chapter.text) > max_tokens:
            sub_parts = split_by_paragraphs(chapter.text, max_tokens)
            for i, part in enumerate(sub_parts):
                final.append(Section(
                    title=f"{chapter.title} (Part {i+1})",
                    text=part,
                    index=len(final)
                ))
        else:
            chapter.index = len(final)
            final.append(chapter)

    # Step 4: Add overlap for continuity
    for i in range(1, len(final)):
        final[i].context_prefix = get_last_n_tokens(
            final[i-1].text, overlap_tokens
        )

    return final

import re

CHAPTER_PATTERNS = [
    r'^#{1,2}\s+.+',                          # Markdown H1/H2
    r'^Chapter\s+\d+',                         # "Chapter 1"
    r'^\d+\.\s+[A-Z]',                        # "1. Introduction"
    r'^PART\s+[IVX]+',                         # "PART III"
    r'^\\(chapter|section)\{',                 # LaTeX commands
    r'^\f',                                    # Form feed (page break)
]

def detect_chapters(text):
    sections = []
    current_title = "Preamble"
    current_start = 0

    for match in re.finditer('|'.join(CHAPTER_PATTERNS), text, re.MULTILINE):
        if match.start() > current_start:
            sections.append(Section(
                title=current_title,
                text=text[current_start:match.start()].strip()
            ))
        current_title = match.group().strip()
        current_start = match.start()

    sections.append(Section(title=current_title, text=text[current_start:].strip()))
    return sections

1. Extract and display Table of Contents
2. Read Introduction (typically Chapter 1)
3. Read Conclusion (typically last chapter)
4. Generate a document map: chapter titles + estimated page counts
5. Identify key themes and arguments

For each chapter:
  - Chapter title and position in document
  - Key arguments or findings (3-5 bullet points)
  - Methodology described (if applicable)
  - Data or evidence presented
  - Connections to previous chapters
  - Open questions or points for follow-up
  - Notable quotes or passages (with page/section references)