#!/usr/bin/env python3
"""
Production-Grade Markdown Formatter
===================================

A comprehensive tool for converting raw conversation logs, txt files, and unstructured 
documentation into professional, well-formatted markdown with proper hierarchy, 
code blocks, and navigation features.

Usage:
    python format-markdown.py input_file.txt [--output output_file.md] [--config config.json]
    python format-markdown.py --batch /path/to/files/*.txt
    python format-markdown.py --help

Features:
- Converts raw conversation logs to structured markdown.
- Extracts and formats code blocks with syntax highlighting
- Generates table of contents with anchor links
- Creates proper heading hierarchy
- Handles various input formats (conversation logs, txt files, etc.)
- Maintains technical accuracy while improving readability
- Supports batch processing of multiple files

Author: Justin Murry
Organization: unparty LLC
Version: v30.0.0
License: MIT
"""

import argparse
import json
import logging
import os
import re
import sys
from datetime import datetime
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union
import hashlib


class MarkdownFormatter:
    """
    Production-grade markdown formatter for conversation logs and txt files.
    
    This class provides comprehensive formatting capabilities to transform
    raw conversation logs, technical documentation, and unstructured text
    into professional markdown documentation.
    """
    
    def __init__(self, config: Optional[Dict] = None):
        """Initialize the markdown formatter with configuration."""
        self.config = config or self._get_default_config()
        self.logger = self._setup_logging()
        self.stats = {
            'original_lines': 0,
            'formatted_lines': 0,
            'code_blocks_found': 0,
            'headers_created': 0,
            'sections_organized': 0
        }
    
    def _get_default_config(self) -> Dict:
        """Get default configuration for the formatter."""
        return {
            'max_line_length': 120,
            'preserve_code_blocks': True,
            'generate_toc': True,
            'auto_headers': True,
            'code_language_detection': True,
            'backup_original': True,
            'output_suffix': '_formatted',
            'log_level': 'INFO',
            'conversation_markers': ['⎿', '⏺', '✻', '>', 'User:', 'Assistant:'],
            'code_indicators': ['```', 'def ', 'function ', 'class ', 'import ', 'from '],
            'section_patterns': [
                r'^[A-Z][A-Z\s]+$',  # All caps sections
                r'^\d+\.\s+',        # Numbered sections
                r'^#+\s',            # Existing headers
                r'^\w+:$'            # Colon-ended titles
            ]
        }
    
    def _setup_logging(self) -> logging.Logger:
        """Setup logging configuration."""
        logger = logging.getLogger('MarkdownFormatter')
        logger.setLevel(getattr(logging, self.config.get('log_level', 'INFO')))
        
        if not logger.handlers:
            handler = logging.StreamHandler()
            formatter = logging.Formatter(
                '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
            )
            handler.setFormatter(formatter)
            logger.addHandler(handler)
        
        return logger
    
    def format_file(self, input_path: Union[str, Path], 
                   output_path: Optional[Union[str, Path]] = None) -> Path:
        """
        Format a single file from raw format to structured markdown.
        
        Args:
            input_path: Path to the input file
            output_path: Optional output path (auto-generated if not provided)
            
        Returns:
            Path to the formatted output file
        """
        input_path = Path(input_path)
        
        if not input_path.exists():
            raise FileNotFoundError(f"Input file not found: {input_path}")
        
        # Generate output path if not provided
        if output_path is None:
            output_path = self._generate_output_path(input_path)
        else:
            output_path = Path(output_path)
        
        self.logger.info(f"Processing file: {input_path}")
        
        # Create backup if configured
        if self.config.get('backup_original', True):
            self._create_backup(input_path)
        
        # Read and process content
        with open(input_path, 'r', encoding='utf-8') as f:
            raw_content = f.read()
        
        self.stats['original_lines'] = len(raw_content.splitlines())
        
        # Format the content
        formatted_content = self._format_content(raw_content, input_path.name)
        
        # Write formatted content
        output_path.parent.mkdir(parents=True, exist_ok=True)
        with open(output_path, 'w', encoding='utf-8') as f:
            f.write(formatted_content)
        
        self.stats['formatted_lines'] = len(formatted_content.splitlines())
        
        self.logger.info(f"Formatted file saved: {output_path}")
        self.logger.info(f"Compression: {self.stats['original_lines']} → {self.stats['formatted_lines']} lines")
        
        return output_path
    
    def batch_format(self, pattern: str) -> List[Path]:
        """
        Format multiple files matching a pattern.
        
        Args:
            pattern: Glob pattern for input files
            
        Returns:
            List of output file paths
        """
        import glob
        input_files = glob.glob(pattern)
        
        if not input_files:
            self.logger.warning(f"No files found matching pattern: {pattern}")
            return []
        
        output_files = []
        for input_file in input_files:
            try:
                output_file = self.format_file(input_file)
                output_files.append(output_file)
            except Exception as e:
                self.logger.error(f"Failed to process {input_file}: {e}")
        
        return output_files
    
    def _format_content(self, content: str, filename: str) -> str:
        """
        Core content formatting logic.
        
        Args:
            content: Raw content to format
            filename: Original filename for context
            
        Returns:
            Formatted markdown content
        """
        lines = content.splitlines()
        formatted_sections = []
        
        # Extract metadata and title
        title, project_info = self._extract_title_and_metadata(lines, filename)
        
        # Parse content into sections
        sections = self._parse_content_sections(lines)
        
        # Build formatted document
        formatted_content = self._build_document(title, project_info, sections)
        
        return formatted_content
    
    def _extract_title_and_metadata(self, lines: List[str], filename: str) -> Tuple[str, Dict]:
        """Extract title and project metadata from content."""
        project_info = {}
        title = "Documentation"
        
        # Look for project patterns in first 20 lines
        for i, line in enumerate(lines[:20]):
            line = line.strip()
            
            # Extract repository name
            if re.match(r'^[A-Z][A-Z_]+$', line) and len(line) > 3:
                project_info['repository'] = line
                title = f"{line} Documentation"
            
            # Extract story/issue ID
            if re.match(r'^S\d+', line) or 'S' in line and any(char.isdigit() for char in line):
                project_info['story_id'] = line
            
            # Extract component from filename or content
            if 'audit_dashboard' in filename or 'audit_dashboard' in line.lower():
                project_info['component'] = 'audit_dashboard_generator.py'
        
        # Use filename as fallback title
        if title == "Documentation":
            title = filename.replace('.txt', '').replace('.md', '').replace('_', ' ').title()
        
        # Add date from filename if present
        date_match = re.search(r'(\d{4}-\d{2}-\d{2})', filename)
        if date_match:
            project_info['date'] = date_match.group(1)
        
        return title, project_info
    
    def _parse_content_sections(self, lines: List[str]) -> List[Dict]:
        """Parse content into logical sections."""
        sections = []
        current_section = None
        current_content = []
        in_code_block = False
        code_language = None
        
        for line_num, line in enumerate(lines):
            stripped = line.strip()
            
            # Handle code blocks
            if stripped.startswith('```'):
                if not in_code_block:
                    # Starting code block
                    in_code_block = True
                    code_language = stripped[3:].strip() or self._detect_language(line)
                    if current_content:
                        current_content.append(f"```{code_language}")
                    else:
                        current_content = [f"```{code_language}"]
                else:
                    # Ending code block
                    in_code_block = False
                    current_content.append("```")
                    self.stats['code_blocks_found'] += 1
                continue
            
            if in_code_block:
                current_content.append(line)
                continue
            
            # Detect section breaks
            if self._is_section_header(stripped) and not in_code_block:
                # Save previous section
                if current_section and current_content:
                    current_section['content'] = '\n'.join(current_content)
                    sections.append(current_section)
                
                # Start new section
                current_section = {
                    'title': self._clean_section_title(stripped),
                    'level': self._determine_header_level(stripped),
                    'type': self._classify_section_type(stripped),
                    'line_number': line_num
                }
                current_content = []
                self.stats['headers_created'] += 1
                continue
            
            # Add content to current section
            if stripped:  # Skip empty lines at section boundaries
                current_content.append(line)
        
        # Add final section
        if current_section and current_content:
            current_section['content'] = '\n'.join(current_content)
            sections.append(current_section)
        
        self.stats['sections_organized'] = len(sections)
        return sections
    
    def _is_section_header(self, line: str) -> bool:
        """Determine if a line should be treated as a section header."""
        if not line:
            return False
        
        # Check against configured patterns
        for pattern in self.config.get('section_patterns', []):
            if re.match(pattern, line):
                return True
        
        # Additional heuristics
        if (line.isupper() and len(line) > 3 and 
            all(c.isalpha() or c.isspace() for c in line)):
            return True
        
        if line.endswith(':') and len(line.split()) <= 3:
            return True
        
        # Conversation markers that indicate new sections
        conversation_markers = self.config.get('conversation_markers', ['⎿', '⏺', '✻'])
        if any(marker in line for marker in conversation_markers):
            return True
        
        return False
    
    def _clean_section_title(self, title: str) -> str:
        """Clean and format section title."""
        # Remove conversation markers
        title = re.sub(r'[⎿⏺✻>]', '', title)
        
        # Remove excessive whitespace
        title = ' '.join(title.split())
        
        # Remove trailing colons
        title = title.rstrip(':')
        
        # Capitalize properly
        if title.isupper():
            title = title.title()
        
        return title.strip()
    
    def _determine_header_level(self, line: str) -> int:
        """Determine appropriate header level for a section."""
        # Existing markdown headers
        if line.startswith('#'):
            return len(line) - len(line.lstrip('#'))
        
        # All caps sections are usually major headers
        if line.isupper():
            return 2
        
        # Numbered sections
        if re.match(r'^\d+\.', line):
            return 3
        
        # Default to h3
        return 3
    
    def _classify_section_type(self, line: str) -> str:
        """Classify the type of section for appropriate formatting."""
        line_lower = line.lower()
        
        if any(word in line_lower for word in ['code', 'implementation', 'script']):
            return 'technical'
        elif any(word in line_lower for word in ['problem', 'issue', 'error']):
            return 'problem'
        elif any(word in line_lower for word in ['solution', 'fix', 'resolve']):
            return 'solution'
        elif any(word in line_lower for word in ['overview', 'summary', 'introduction']):
            return 'overview'
        else:
            return 'general'
    
    def _detect_language(self, line: str) -> str:
        """Detect programming language from code context."""
        content = line.lower()
        
        if any(kw in content for kw in ['def ', 'import ', 'class ', 'python']):
            return 'python'
        elif any(kw in content for kw in ['function', 'var ', 'const ', 'javascript']):
            return 'javascript'
        elif any(kw in content for kw in ['<div', '<html', 'html']):
            return 'html'
        elif any(kw in content for kw in ['{', '}', 'css']):
            return 'css'
        elif 'bash' in content or 'shell' in content:
            return 'bash'
        else:
            return ''
    
    def _build_document(self, title: str, project_info: Dict, sections: List[Dict]) -> str:
        """Build the final formatted markdown document."""
        document_parts = []
        
        # Document header
        document_parts.append(f"# {title}\n")
        
        # Project metadata
        if project_info:
            document_parts.append("## Project Context\n")
            for key, value in project_info.items():
                formatted_key = key.replace('_', ' ').title()
                document_parts.append(f"**{formatted_key}:** `{value}`  ")
            document_parts.append("")
        
        # Overview section
        document_parts.append("## Overview\n")
        document_parts.append(
            "This document provides comprehensive documentation extracted and formatted "
            "from development conversation logs and technical analysis.\n"
        )
        
        # Table of contents
        if self.config.get('generate_toc', True) and sections:
            document_parts.append("## Table of Contents\n")
            for i, section in enumerate(sections, 1):
                anchor = self._create_anchor(section['title'])
                document_parts.append(f"{i}. [{section['title']}](#{anchor})")
            document_parts.append("\n---\n")
        
        # Sections
        for section in sections:
            header_level = '#' * section['level']
            document_parts.append(f"{header_level} {section['title']}\n")
            
            # Add section content with proper formatting
            content = self._format_section_content(section['content'], section['type'])
            document_parts.append(content)
            document_parts.append("")  # Add spacing between sections
        
        # Document footer
        document_parts.append("---\n")
        document_parts.append(f"*Document generated by format-markdown.py on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*")
        
        return '\n'.join(document_parts)
    
    def _format_section_content(self, content: str, section_type: str) -> str:
        """Format section content based on its type."""
        if not content:
            return ""
        
        lines = content.split('\n')
        formatted_lines = []
        
        for line in lines:
            # Clean up conversation markers
            cleaned_line = re.sub(r'^[⎿⏺✻]\s*', '', line)
            
            # Format code snippets that aren't in code blocks
            code_indicators = self.config.get('code_indicators', ['```', 'def ', 'function ', 'class ', 'import ', 'from '])
            if (not line.strip().startswith('```') and 
                any(indicator in line for indicator in code_indicators)):
                if not any(l.strip().startswith('```') for l in lines):  # Not already in code block
                    language = self._detect_language(line)
                    formatted_lines.append(f"```{language}")
                    formatted_lines.append(cleaned_line)
                    formatted_lines.append("```")
                    continue
            
            # Add emphasis to important statements
            if section_type == 'problem' and line.strip():
                if not line.startswith('>'):
                    cleaned_line = f"> {cleaned_line}"
            
            formatted_lines.append(cleaned_line)
        
        return '\n'.join(formatted_lines)
    
    def _create_anchor(self, title: str) -> str:
        """Create URL-safe anchor for table of contents."""
        anchor = title.lower()
        anchor = re.sub(r'[^\w\s-]', '', anchor)
        anchor = re.sub(r'[-\s]+', '-', anchor)
        return anchor.strip('-')
    
    def _generate_output_path(self, input_path: Path) -> Path:
        """Generate output path for formatted file."""
        suffix = self.config.get('output_suffix', '_formatted')
        
        if input_path.suffix == '.md':
            # If already markdown, add suffix before extension
            stem = input_path.stem
            if not stem.endswith(suffix):
                return input_path.with_name(f"{stem}{suffix}.md")
            return input_path
        else:
            # Convert to markdown
            return input_path.with_suffix('.md')
    
    def _create_backup(self, input_path: Path) -> Path:
        """Create backup of original file."""
        backup_path = input_path.with_suffix(input_path.suffix + '.backup')
        
        if not backup_path.exists():
            import shutil
            shutil.copy2(input_path, backup_path)
            self.logger.info(f"Backup created: {backup_path}")
        
        return backup_path
    
    def get_stats(self) -> Dict:
        """Get formatting stats."""
        return self.stats.copy()


def main():
    """Main entry point for the markdown formatter."""
    parser = argparse.ArgumentParser(
        description="Production-grade markdown formatter for conversation logs and txt files",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__
    )
    
    parser.add_argument(
        'input_file',
        nargs='?',
        help='Input file to format (txt, md, or conversation log)'
    )
    
    parser.add_argument(
        '--output', '-o',
        help='Output file path (auto-generated if not specified)'
    )
    
    parser.add_argument(
        '--batch', '-b',
        help='Process multiple files using glob pattern (e.g., "*.txt")'
    )
    
    parser.add_argument(
        '--config', '-c',
        help='Configuration file (JSON format)'
    )
    
    parser.add_argument(
        '--log-level',
        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
        default='INFO',
        help='Logging level'
    )
    
    parser.add_argument(
        '--stats',
        action='store_true',
        help='Display formatting statistics'
    )
    
    parser.add_argument(
        '--version',
        action='version',
        version='format-markdown.py 1.0.0'
    )
    
    args = parser.parse_args()
    
    # Load configuration
    config = {}
    if args.config and os.path.exists(args.config):
        with open(args.config, 'r') as f:
            config = json.load(f)
    
    config['log_level'] = args.log_level
    
    # Initialize formatter
    formatter = MarkdownFormatter(config)
    
    try:
        if args.batch:
            # Batch processing
            output_files = formatter.batch_format(args.batch)
            print(f"Processed {len(output_files)} files")
            for output_file in output_files:
                print(f"  → {output_file}")
        
        elif args.input_file:
            # Single file processing
            output_file = formatter.format_file(args.input_file, args.output)
            print(f"Formatted: {args.input_file} → {output_file}")
        
        else:
            # No input specified, show help
            parser.print_help()
            return 1
        
        # Display statistics if requested
        if args.stats:
            stats = formatter.get_stats()
            print("\nFormatting Statistics:")
            for key, value in stats.items():
                print(f"  {key.replace('_', ' ').title()}: {value}")
    
    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        return 1
    
    return 0


if __name__ == '__main__':
    sys.exit(main())