origin/sm/sft

lib/style-transfer/bash/experiment/rb_gen/az/llama3.2-3b-complete-replaced.sh

@@ -0,0 +1,15 @@
+export CUDA_VISIBLE_DEVICES=$1
+python style_transfer/run_rb_gen.py model.name=meta-llama/Llama-3.2-3B-Instruct \
+            model.peft_config.target_modules='["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]' \
+            dataset.name=bio-datasets/mimic-iii-gpt4o-tokens \
+            max_steps=5 \
+            dataset.num_generated_samples=3500 \
+            score.model.model_name_or_path=sentence-transformers/all-mpnet-base-v2 \
+            dataset.sft_ratio=0.06 \
+            dataset.gen_ratio=0.7 \
+            dataset.sft_dataset=null \
+            sft.training_args.eval_steps=30 \
+            score.train.train_size=0.6 \
+            dpo.training_args.num_train_epochs=10 \
+            dpo.percentile=70 \
+            score.batch_size=8

lib/style-transfer/bash/experiment/rb_gen/az/llama3.2-3b-pb-seed-replaced.sh

@@ -0,0 +1,15 @@
+export CUDA_VISIBLE_DEVICES=$1
+python style_transfer/run_rb_gen.py model.name=meta-llama/Llama-3.2-3B-Instruct \
+            model.peft_config.target_modules='["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]' \
+            dataset.name=bio-datasets/mimic-iii-gpt4o-tokens \
+            max_steps=5 \
+            dataset.num_generated_samples=3500 \
+            score.model.model_name_or_path=sentence-transformers/all-mpnet-base-v2 \
+            dataset.sft_ratio=0.06 \
+            dataset.gen_ratio=0.7 \
+            sft.training_args.eval_steps=30 \
+            score.train.train_size=0.6 \
+            dataset.sft_dataset.size=977 \
+            dpo.training_args.num_train_epochs=20 \
+            dpo.percentile=70 \
+            score.batch_size=8

lib/style-transfer/style_transfer/rb_gen/steps/sft.py

@@ -60,10 +60,11 @@ def sft_train(
 
     cfg.sft.training_args.output_dir = f"models/{wandb.run.id}/sft"
     args = hydra.utils.instantiate(cfg.sft.training_args)
+    wandb.config.update({"state": "sft"}, allow_val_change=True)
     test_sft_dataset = None
     if cfg.dataset.sft_dataset is not None:
         sft_dataset, test_sft_dataset = sft_dataset.train_test_split(
-            train_size=0.1, shuffle=False
+            train_size=0.5, shuffle=False
         ).values()
     args.load_best_model_at_end = True
     trainer = SFTTrainer(

lib/style-transfer/style_transfer/rb_gen/utils/utils.py

@@ -61,6 +61,10 @@ def build_dataset(
 
     ds_dict: dict = {"keywords": [], "text": []}
     for keywords, text in zip(ds["keywords"], ds["text"]):
+        if isinstance(keywords, str):
+            keywords = [keywords]
+        if isinstance(text, str):
+            text = [text]
         for kw, t in zip(keywords, text):
             ds_dict["keywords"].append(kw)
             ds_dict["text"].append(t)