used path.join instead of adding strings

needlehaystack/llm_needle_haystack_tester.py

@@ -284,9 +284,10 @@ def get_context_length_in_tokens(self, context):
     def read_context_files(self):
         context = ""
         max_context_length = max(self.context_lengths)
+        base_dir = os.path.abspath(os.path.dirname(__file__))  # Package directory
 
         while self.get_context_length_in_tokens(context) < max_context_length:
-            for file in glob.glob(f"{self.haystack_dir}/*.txt"):
+            for file in glob.glob(os.path.join(base_dir, self.haystack_dir, "*.txt")):
                 with open(file, 'r') as f:
                     context += f.read()
         return context

needlehaystack/run.py

@@ -17,7 +17,7 @@ class CommandArgs():
     model_name: str = "gpt-3.5-turbo-0125"
     evaluator_model_name: Optional[str] = "gpt-3.5-turbo-0125"
     needle: Optional[str] = "\nThe best thing to do in San Francisco is eat a sandwich and sit in Dolores Park on a sunny day.\n"
-    haystack_dir: Optional[str] = "needlehaystack/PaulGrahamEssays"
+    haystack_dir: Optional[str] = "PaulGrahamEssays"
     retrieval_question: Optional[str] = "What is the best thing to do in San Francisco?"
     results_version: Optional[int] = 1
     context_lengths_min: Optional[int] = 1000