chore: refactor the code, remove data files

arkhn · Oct 31, 2024 · 034c49b · 034c49b
1 parent 95a4bd6
commit 034c49b
Show file tree

Hide file tree

Showing 5 changed files with 4 additions and 6,770 deletions.
diff --git a/lib/questions_eval/bash/experiments/tiny_mimoracle.sh b/lib/questions_eval/bash/experiments/tiny_mimoracle.sh
@@ -1,4 +1,3 @@
-# python run_mimoracle.py -m model=gpt-4o samples=18 num_questions=5
-# python run_mimoracle.py -m model=gpt-4o-mini samples=18 num_questions=5
-# python run_mimoracle.py -m model=llama3.1-405b-local samples=18 num_questions=5
-python run_mimoracle.py -m model=gpt-4o samples=2 num_questions=2
+python run_mimoracle.py -m model=gpt-4o samples=18 num_questions=5
+python run_mimoracle.py -m model=gpt-4o-mini samples=18 num_questions=5
+python run_mimoracle.py -m model=llama3.1-405b-local samples=18 num_questions=5
diff --git a/lib/questions_eval/hf_datasets/sub_mimoracle/_preprocessing.py b/lib/questions_eval/hf_datasets/sub_mimoracle/_preprocessing.py
@@ -13,7 +13,7 @@ def _preprocess(text: str) -> str:
 def _resample(df: pd.DataFrame, n_sample: int, n_section: int) -> pd.DataFrame:
     patterns = "allergies|history of present illness|past medical history|\
                 discharge medications|social history|medications on admission"
-    df["section_title"] = [_preprocess(x) for x in df["section_title"]]
+    df["section_title"] = df["section_title"].apply(_preprocess)
     df = df[df.section_title.str.contains(patterns)]
     df = df.groupby("section_title").filter(lambda x: len(x) > n_sample)
     df = df.groupby("document_id").filter(lambda x: len(x) == n_section)