Initial commit: SheepOp LLM - Transformer-based language model implementation

- Complete transformer implementation from scratch
- Training pipeline with gradient accumulation and mixed precision
- Optimized inference with KV caching
- Multi-format data processing (PDFs, images, code, text)
- Comprehensive documentation
- Apache 2.0 license
- Example training plots included in docs/images/
This commit is contained in:
Carlos Gutierrez
2025-11-06 22:07:41 -05:00
commit 3d2da94ce2
60 changed files with 25153 additions and 0 deletions

35
models/__init__.py Normal file
View File

@@ -0,0 +1,35 @@
"""
SheepOp LLM - A modern language model implementation
Optimized for production RAG systems
"""
from .transformer import TransformerModel
from .attention import MultiHeadAttention, PositionalEncoding
from .blocks import TransformerBlock, FeedForward
from .optimized_attention import (
OptimizedMultiHeadAttention,
RetrievalCache,
OptimizedInference,
KVCache,
)
from .prefetching import (
PrefetchDataLoader,
LookaheadRetriever,
BatchPrefetcher,
)
__all__ = [
'TransformerModel',
'MultiHeadAttention',
'PositionalEncoding',
'TransformerBlock',
'FeedForward',
'OptimizedMultiHeadAttention',
'RetrievalCache',
'OptimizedInference',
'KVCache',
'PrefetchDataLoader',
'LookaheadRetriever',
'BatchPrefetcher',
]