feat:fix the multiple files issue and add pagination logic to the table

keep-ui/app/runbooks/models.ts

@@ -14,3 +14,8 @@ export interface RunbookDto {
   repo_id: string;
   file_path: string;
 }
+
+export type RunbookResponse = {
+  runbooks: RunbookDto[];
+  total_count: number;
+};

keep-ui/app/runbooks/runbook-table.tsx

@@ -26,7 +26,7 @@ import useSWR from "swr";
 import { fetcher } from "@/utils/fetcher";
 import { useSession } from "next-auth/react";
 import RunbookActions from "./runbook-actions";
-import { RunbookDto } from "./models";
+import { RunbookDto, RunbookResponse } from "./models";
 
 const customStyles = {
   content: {
@@ -71,13 +71,20 @@ const columnsv2 = [
     id: "contents",
     header: "Contents",
     cell: ({ row }) => {
+      const contents = row.original.contents || [];
+      const isMoreContentAvailable = contents.length > 4;
       return (
         <div>
-          {row.original.contents?.map((content: Content) => (
+          {contents.slice(0, 4)?.map((content: Content) => (
             <Badge key={content.id} color="green" className="mr-2 mb-1">
               {content.file_name}
             </Badge>
           ))}
+          {isMoreContentAvailable && (
+            <Badge color="green" className="mr-2 mb-1">{`${
+              contents.length - 4
+            } more...`}</Badge>
+          )}
         </div>
       );
     },
@@ -262,13 +269,20 @@ function RunbookIncidentTable() {
 
   let shouldFetch = session?.accessToken ? true : false;
 
-  const { data: runbooksData, error } = useSWR<RunbookDto[]>(
-    shouldFetch ? `${getApiURL()}/runbooks` : null,
+  const { data: runbooksData, error } = useSWR<RunbookResponse>(
+    shouldFetch
+      ? `${getApiURL()}/runbooks?limit=${limit}&offset=${offset}`
+      : null,
     (url: string) => {
       return fetcher(url, session?.accessToken!);
     }
   );
 
+  const { total_count, runbooks } = runbooksData || {
+    total_count: 0,
+    runbooks: [],
+  };
+
   // Modal state management
 
   const handlePaginationChange = (newLimit: number, newOffset: number) => {
@@ -280,7 +294,7 @@ function RunbookIncidentTable() {
     return (
       <RunbookActions
         selectedRowIds={selectedRowIds}
-        runbooks={runbooksData || []}
+        runbooks={runbooks || []}
         clearRowSelection={table.resetRowSelection}
       />
     );
@@ -293,11 +307,11 @@ function RunbookIncidentTable() {
         <SettingsPage />
       </div>
       <Card className="flex-1 overflow-auto">
-        {runbooksData && (
+        {!!total_count && (
           <GenericTable<RunbookDto>
-            data={runbooksData}
+            data={runbooks}
             columns={columnsv2}
-            rowCount={runbooksData.length}
+            rowCount={total_count}
             offset={offset}
             limit={limit}
             onPaginationChange={handlePaginationChange}

keep/api/routes/runbooks.py

@@ -118,16 +118,16 @@ def create_runbook(
 @router.get(
     "",
     description="All Runbooks",
-    # response_model=RunbookDtoOut,
 )
-def create_runbook(
-
+def get_all_runbooks(
+    limit: int = 25,
+    offset: int = 0,
     authenticated_entity: AuthenticatedEntity = Depends(
         IdentityManagerFactory.get_auth_verifier(["read:runbook"])
     ),
     session: Session = Depends(get_session)
 ):
     tenant_id = authenticated_entity.tenant_id
     logger.info("get all Runbooks", extra={tenant_id: tenant_id})
-    return RunbookService.get_all_runbooks(session, tenant_id)    
+    return RunbookService.get_all_runbooks(session, tenant_id, limit, offset)    
 

keep/providers/github_provider/github_provider.py

@@ -122,31 +122,52 @@ def pull_repositories(self, project_id=None):
         repos_list = self._format_repos(repos)
         return repos_list
 
-    def _format_runbook(self, runbook, repo, title):
+
+    def _format_content(self, runbookContent, repo):
+        """
+        Format the content data into a dictionary.
+        """
+        return {
+            "content": runbookContent.content,
+            "link": f"https://api.github.com/{repo.get('full_name')}/blob/{repo.get('default_branch')}/{runbookContent.path}",
+            "encoding": runbookContent.encoding,
+            "file_name": runbookContent.name
+        }    
+
+    def _format_runbook(self, runbook, repo, title, md_path):
         """
         Format the runbook data into a dictionary.
         """
-
-        # TO DO. currently we are handling the one file only. we user give folder path. then we might get multiple files as input(runbook)
+        self.logger.info("runbook: %s", runbook)
+
+        if runbook is None:
+            raise Exception("Got empty runbook. Please check the runbook path and try again.")
+
+        # Check if the runbook is a list, then set runbook_contents accordingly
+        if isinstance(runbook, list):
+            runbook_contents = runbook           
+        else:
+            runbook_contents = [runbook] 
+
+        # Filter contents where type is "file"
+        filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents if runbookContent.type == "file"]
+
+        # Format the contents using a helper function
+        contents = [self._format_content(runbookContent, repo) for runbookContent in filtered_runbook_contents]
+
+        # Return the formatted runbook data as a dictionary
         return {
-            "file_name": runbook.name,
-            "file_path": runbook.path,
-            "file_size": runbook.size,
-            "file_type": runbook.type,
+            "relative_path": md_path,
             "repo_id": repo.get("id"),
             "repo_name": repo.get("name"),
             "repo_display_name": repo.get("display_name"),
             "provider_type": "github",
             "provider_id": self.provider_id,
-            "contents": [{
-                "content":runbook.content,
-                "link": f"https://api.github.com/{repo.get('full_name')}/blob/{repo.get('default_branch')}/{runbook.path}",
-                "encoding": runbook.encoding,
-                "file_name": runbook.name
-                }],
+            "contents": contents,
             "title": title,
         }
 
+
 
 
     def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): 
@@ -170,7 +191,7 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None):
                     raise Exception(f"Repository {repo_name} not found")
 
                 runbook = repo.get_contents(md_path, branch)
-                response = self._format_runbook(runbook, self._format_repo(repo), title)
+                response = self._format_runbook(runbook, self._format_repo(repo), title, md_path)
                 return response
 
             except GithubException as e:

keep/providers/gitlab_provider/gitlab_provider.py

@@ -228,29 +228,49 @@ def pull_repositories(self, project_id=None):
 
        raise Exception("Failed to get repositories: personal_access_token not set")
 
-    def _format_runbook(self, runbook, repo, title):
-       """
-       Format the runbook data into a dictionary.
-       """
-
-       # TO DO. currently we are handling the one file only. we user give folder path. then we might get multiple files as input(runbook)
-       return  {
-                "file_name": runbook.get("file_name"),
-                "file_path": runbook.get("file_path"),
-                "file_size": runbook.get("size"),
-                "file_type": runbook.get("type"),
-                "repo_id": repo.get("id"),
-                "repo_name": repo.get("name"),
-                "repo_display_name": repo.get("display_name"),
-                "provider_type": "gitlab",
-                "config": self.provider_id,
-                "contents": [{
-                    "content": runbook.get("content"),
-                    "link": f"{self.gitlab_host}/api/v4/projects/{repo.get('id')}/repository/files/{runbook.get('file_path')}/raw",
-                    "encoding": runbook.get("encoding"),
-                    }],
-                "title": title,
-                }           
+    def _format_content(self, runbookContent, repo):
+        """
+        Format the content data into a dictionary.
+        """
+        return {
+            "content": runbookContent.get("content"),
+            "link": f"{self.gitlab_host}/api/v4/projects/{repo.get('id')}/repository/files/{runbookContent.get('file_path')}/raw",
+            "encoding": runbookContent.get("encoding"),
+            "file_name": runbookContent.get("file_name"),
+        }
+
+
+    def _format_runbook(self, runbook, repo, title, md_path):
+        """
+        Format the runbook data into a dictionary.
+        """
+        if runbook is None:
+            raise Exception("Got empty runbook. Please check the runbook path and try again.")
+
+        # Check if runbook is a list, if not convert to list
+        if isinstance(runbook, list):
+            runbook_contents = runbook      
+        else:
+            runbook_contents = [runbook] 
+
+        # Filter runbook contents where type is "file"
+        filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents]
+
+        # Format the contents using a helper function
+        contents = [self._format_content(runbookContent, repo) for runbookContent in filtered_runbook_contents]
+
+        # Return formatted runbook data as dictionary
+        return {
+            "relative_path": md_path,
+            "repo_id": repo.get("id"),
+            "repo_name": repo.get("name"),
+            "repo_display_name": repo.get("display_name"),
+            "provider_type": "gitlab",  # This was changed from "github" to "gitlab", assuming it is intentional
+            "provider_id": self.provider_id,  # Assuming this is supposed to be 'provider_id', not 'config'
+            "contents": contents,
+            "title": title,
+        }
+
 
     def pull_runbook(self, repo=None, branch=None, md_path=None, title=None):
         """Retrieve markdown files from the GitLab repository."""
@@ -259,9 +279,8 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None):
         md_path = md_path if md_path else self.authentication_config.md_path
 
         repo_meta = self.pull_repositories(project_id=repo)
-
-        if repo_meta and branch and md_path:
-            repo_id = repo_meta.get("id")
+        repo_id = repo_meta.get("id")
+        if repo_id and branch and md_path:
             resp = requests.get(
                 f"{self.gitlab_host}/api/v4/projects/{repo_id}/repository/files/{md_path}?ref={branch}",
                 headers=self.__get_auth_header()
@@ -272,7 +291,7 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None):
             except HTTPError as e:
                 raise Exception(f"Failed to get runbook: {e}")
 
-            return self._format_runbook(resp.json(), repo_meta, title)
+            return self._format_runbook(resp.json(), repo_meta, title, md_path)
 
         raise Exception("Failed to get runbook: repository or md_path not set")       
 

keep/runbooks/runbooks_service.py

@@ -1,9 +1,8 @@
 import logging
-from typing import List
 from pydantic import ValidationError
 from sqlalchemy.orm import  selectinload
 
-from sqlmodel import Session, select
+from sqlmodel import Session
 from keep.api.models.db.runbook import (
     Runbook,
     RunbookContent,
@@ -20,7 +19,7 @@ def create_runbook(session: Session, tenant_id: str, runbook_dto: dict):
                 tenant_id=tenant_id,
                 title=runbook_dto["title"],
                 repo_id=runbook_dto["repo_id"],
-                relative_path=runbook_dto["file_path"],
+                relative_path=runbook_dto["relative_path"],
                 provider_type=runbook_dto["provider_type"],
                 provider_id=runbook_dto["provider_id"]
             )
@@ -50,11 +49,14 @@ def create_runbook(session: Session, tenant_id: str, runbook_dto: dict):
             logger.exception(f"Failed to create runbook {e}")
 
     @staticmethod
-    def get_all_runbooks(session: Session, tenant_id: str) -> List[RunbookDtoOut]:
-        runbooks = session.exec(
-            select(Runbook)
-            .where(Runbook.tenant_id == tenant_id)
-            .options(selectinload(Runbook.contents)).limit(1000)
+    def get_all_runbooks(session: Session, tenant_id: str, limit=25, offset=0) -> dict:
+        query = session.query(Runbook).filter(
+            Runbook.tenant_id == tenant_id,
         )
 
-        return [RunbookDtoOut.from_orm(runbook) for runbook in runbooks]            
+        total_count = query.count()  # Get the total count of runbooks matching the tenant_id
+        runbooks = query.options(selectinload(Runbook.contents)).limit(limit).offset(offset).all()  # Fetch the paginated runbooks
+        result = [RunbookDtoOut.from_orm(runbook) for runbook in runbooks]  # Convert runbooks to DTOs
+
+        # Return total count and list of runbooks
+        return {"total_count": total_count, "runbooks": result}