do not land: local setup

Summary: Test Plan:
determined-ai · Jul 31, 2024 · ad8091a · ad8091a
1 parent 8248344
commit ad8091a
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 4 deletions.
diff --git a/blog/llm-finetuning-4/finetune.py b/blog/llm-finetuning-4/finetune.py
@@ -6,6 +6,7 @@
 import evaluate
 import torch
 import transformers
+import wandb
 from determined.transformers import DetCallback
 from peft import AutoPeftModelForCausalLM, LoraConfig, get_peft_model
 from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments
@@ -173,6 +174,29 @@ def compute_metrics(eval_preds):
         distributed = det.core.DistributedContext.from_torch_distributed()
 
     with det.core.init(distributed=distributed) as core_context:
+        if core_context.distributed.rank == 0:
+            wandb.login(key=hparams["wandb_key"])
+            import uuid
+            # Generate a UUID
+            my_uuid = uuid.uuid4()
+            # Convert UUID to string
+            uuid_str = str(my_uuid)[:5]
+            r = hparams["r"]
+            lora_alpha = hparams["lora_alpha_in_r"]
+            lora_dropout = hparams["lora_dropout"]
+            dataset_subset = hparams["dataset_subset"]
+            run_name = f"test_lora_blog_{dataset_subset}_r_{r}_alpha_in_r_{lora_alpha}_dropout_{lora_dropout}_{uuid_str}"
+            run = wandb.init(
+                project="lora-blog", 
+                name=run_name, 
+                config={
+                    "r":hparams["r"],
+                    "lora_alpha_in_r":hparams["lora_alpha_in_r"],
+                    "dropout":hparams["lora_dropout"],
+                    "dataset_subset":hparams["dataset_subset"],
+                    "model":hparams["model"]
+                }
+            )
         det_callback = DetCallback(
             core_context,
             training_args,

diff --git a/blog/llm-finetuning-4/lora.yaml b/blog/llm-finetuning-4/lora.yaml
@@ -1,4 +1,4 @@
-name: mistral lora easy
+name: mistral lora hard
 debug: false
 environment:
   environment_variables:
@@ -7,7 +7,10 @@ environment:
     gpu: determinedai/environments:cuda-11.8-pytorch-2.0-gpu-95c7a14
     cpu: determinedai/environments:py-3.10-pytorch-2.0-cpu-03ae7d7
 resources:
-  slots_per_trial: 2
+  slots_per_trial: 4
+  resource_pool: a100
+workspace: swy_5
+project: lora-blog
 searcher:
   name: grid
   max_length:
@@ -17,7 +20,7 @@ searcher:
 hyperparameters:
   model: "mistralai/Mistral-7B-Instruct-v0.2"
   model_commit_hash: "99259002b41e116d28ccb2d04a9fbe22baed0c7f"
-  dataset_subset: "easy"
+  dataset_subset: "hard"
   lora: true
   # Tunable hyperparameters
   r:
@@ -30,10 +33,12 @@ hyperparameters:
     type: categorical
     vals: [0.1]
   # End tunable hyperparameters
+  hf_token: HF_TOKEN
+  wandb_key: WB_KEY
   training_args:
     output_dir: "/tmp/llm_finetuning"
     max_steps: 5000
-    per_device_train_batch_size: 8
+    per_device_train_batch_size: 4
     per_device_eval_batch_size: 4
     bf16: true
     evaluation_strategy: "steps"
@@ -43,6 +48,7 @@ hyperparameters:
     save_strategy: "steps"
     save_steps: 1000
     learning_rate: 1e-5
+    report_to: "wandb"
 entrypoint: >-
   python -m determined.launch.torch_distributed
   python finetune.py

diff --git a/blog/llm-finetuning-4/startup-hook.sh b/blog/llm-finetuning-4/startup-hook.sh
@@ -1,3 +1,5 @@
 #!/bin/bash
 pip install --upgrade pip
 pip install -r requirements.txt
+# local requirements
+pip install wandb