Experience reading, reimagined.

# Initialize model and tokenizer
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM

model_name = "deepseek-ai/deepseek-coder-6.7b-base"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)

# Load custom dataset
dataset = load_dataset("rij-ai/custom-instructions")
train_data = dataset["train"]

# Training configuration
training_args = TrainingArguments(
    output_dir="./rij-ai-finetuned",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-5,
    fp16=True,
    save_steps=500,
    logging_steps=100,
)

# Initialize trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_data,
    data_collator=DataCollatorForLanguageModeling(
        tokenizer=tokenizer,
        mlm=False
    )
)

# Start training process
print("Starting fine-tuning process...")
trainer.train()

# Save the fine-tuned model
trainer.save_model("./rij-ai-finetuned-final")
print("Fine-tuning complete! Model saved.")