From dd57ffbd05ba1529d3087aa89df7adf032a49f42 Mon Sep 17 00:00:00 2001 From: Rajesh Jonnalagadda Date: Wed, 9 Oct 2024 21:12:08 +0530 Subject: [PATCH 1/3] feat:fix the multiple files issue and add pagination logic to the table --- keep-ui/app/runbooks/models.ts | 5 ++ keep-ui/app/runbooks/runbook-table.tsx | 30 +++++--- keep/api/routes/runbooks.py | 8 +-- .../github_provider/github_provider.py | 49 +++++++++---- .../gitlab_provider/gitlab_provider.py | 68 ++++++++++++------- keep/runbooks/runbooks_service.py | 19 +++--- 6 files changed, 121 insertions(+), 58 deletions(-) diff --git a/keep-ui/app/runbooks/models.ts b/keep-ui/app/runbooks/models.ts index fbb82cfb0..187698ccd 100644 --- a/keep-ui/app/runbooks/models.ts +++ b/keep-ui/app/runbooks/models.ts @@ -14,3 +14,8 @@ export interface RunbookDto { repo_id: string; file_path: string; } + +export type RunbookResponse = { + runbooks: RunbookDto[]; + total_count: number; +}; diff --git a/keep-ui/app/runbooks/runbook-table.tsx b/keep-ui/app/runbooks/runbook-table.tsx index db4b31ee3..7a83746fd 100644 --- a/keep-ui/app/runbooks/runbook-table.tsx +++ b/keep-ui/app/runbooks/runbook-table.tsx @@ -26,7 +26,7 @@ import useSWR from "swr"; import { fetcher } from "@/utils/fetcher"; import { useSession } from "next-auth/react"; import RunbookActions from "./runbook-actions"; -import { RunbookDto } from "./models"; +import { RunbookDto, RunbookResponse } from "./models"; const customStyles = { content: { @@ -71,13 +71,20 @@ const columnsv2 = [ id: "contents", header: "Contents", cell: ({ row }) => { + const contents = row.original.contents || []; + const isMoreContentAvailable = contents.length > 4; return (
- {row.original.contents?.map((content: Content) => ( + {contents.slice(0, 4)?.map((content: Content) => ( {content.file_name} ))} + {isMoreContentAvailable && ( + {`${ + contents.length - 4 + } more...`} + )}
); }, @@ -262,13 +269,20 @@ function RunbookIncidentTable() { let shouldFetch = session?.accessToken ? true : false; - const { data: runbooksData, error } = useSWR( - shouldFetch ? `${getApiURL()}/runbooks` : null, + const { data: runbooksData, error } = useSWR( + shouldFetch + ? `${getApiURL()}/runbooks?limit=${limit}&offset=${offset}` + : null, (url: string) => { return fetcher(url, session?.accessToken!); } ); + const { total_count, runbooks } = runbooksData || { + total_count: 0, + runbooks: [], + }; + // Modal state management const handlePaginationChange = (newLimit: number, newOffset: number) => { @@ -280,7 +294,7 @@ function RunbookIncidentTable() { return ( ); @@ -293,11 +307,11 @@ function RunbookIncidentTable() { - {runbooksData && ( + {!!total_count && ( - data={runbooksData} + data={runbooks} columns={columnsv2} - rowCount={runbooksData.length} + rowCount={total_count} offset={offset} limit={limit} onPaginationChange={handlePaginationChange} diff --git a/keep/api/routes/runbooks.py b/keep/api/routes/runbooks.py index 17c631cbd..86cdbb578 100644 --- a/keep/api/routes/runbooks.py +++ b/keep/api/routes/runbooks.py @@ -118,10 +118,10 @@ def create_runbook( @router.get( "", description="All Runbooks", - # response_model=RunbookDtoOut, ) -def create_runbook( - +def get_all_runbooks( + limit: int = 25, + offset: int = 0, authenticated_entity: AuthenticatedEntity = Depends( IdentityManagerFactory.get_auth_verifier(["read:runbook"]) ), @@ -129,5 +129,5 @@ def create_runbook( ): tenant_id = authenticated_entity.tenant_id logger.info("get all Runbooks", extra={tenant_id: tenant_id}) - return RunbookService.get_all_runbooks(session, tenant_id) + return RunbookService.get_all_runbooks(session, tenant_id, limit, offset) diff --git a/keep/providers/github_provider/github_provider.py b/keep/providers/github_provider/github_provider.py index 4f2b628ae..76f6ad0d1 100644 --- a/keep/providers/github_provider/github_provider.py +++ b/keep/providers/github_provider/github_provider.py @@ -122,31 +122,52 @@ def pull_repositories(self, project_id=None): repos_list = self._format_repos(repos) return repos_list - def _format_runbook(self, runbook, repo, title): + + def _format_content(self, runbookContent, repo): + """ + Format the content data into a dictionary. + """ + return { + "content": runbookContent.content, + "link": f"https://api.github.com/{repo.get('full_name')}/blob/{repo.get('default_branch')}/{runbookContent.path}", + "encoding": runbookContent.encoding, + "file_name": runbookContent.name + } + + def _format_runbook(self, runbook, repo, title, md_path): """ Format the runbook data into a dictionary. """ - - # TO DO. currently we are handling the one file only. we user give folder path. then we might get multiple files as input(runbook) + self.logger.info("runbook: %s", runbook) + + if runbook is None: + raise Exception("Got empty runbook. Please check the runbook path and try again.") + + # Check if the runbook is a list, then set runbook_contents accordingly + if isinstance(runbook, list): + runbook_contents = runbook + else: + runbook_contents = [runbook] + + # Filter contents where type is "file" + filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents if runbookContent.type == "file"] + + # Format the contents using a helper function + contents = [self._format_content(runbookContent, repo) for runbookContent in filtered_runbook_contents] + + # Return the formatted runbook data as a dictionary return { - "file_name": runbook.name, - "file_path": runbook.path, - "file_size": runbook.size, - "file_type": runbook.type, + "relative_path": md_path, "repo_id": repo.get("id"), "repo_name": repo.get("name"), "repo_display_name": repo.get("display_name"), "provider_type": "github", "provider_id": self.provider_id, - "contents": [{ - "content":runbook.content, - "link": f"https://api.github.com/{repo.get('full_name')}/blob/{repo.get('default_branch')}/{runbook.path}", - "encoding": runbook.encoding, - "file_name": runbook.name - }], + "contents": contents, "title": title, } + def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): @@ -170,7 +191,7 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): raise Exception(f"Repository {repo_name} not found") runbook = repo.get_contents(md_path, branch) - response = self._format_runbook(runbook, self._format_repo(repo), title) + response = self._format_runbook(runbook, self._format_repo(repo), title, md_path) return response except GithubException as e: diff --git a/keep/providers/gitlab_provider/gitlab_provider.py b/keep/providers/gitlab_provider/gitlab_provider.py index 563195b80..50c776818 100644 --- a/keep/providers/gitlab_provider/gitlab_provider.py +++ b/keep/providers/gitlab_provider/gitlab_provider.py @@ -228,29 +228,49 @@ def pull_repositories(self, project_id=None): raise Exception("Failed to get repositories: personal_access_token not set") - def _format_runbook(self, runbook, repo, title): - """ - Format the runbook data into a dictionary. - """ - - # TO DO. currently we are handling the one file only. we user give folder path. then we might get multiple files as input(runbook) - return { - "file_name": runbook.get("file_name"), - "file_path": runbook.get("file_path"), - "file_size": runbook.get("size"), - "file_type": runbook.get("type"), - "repo_id": repo.get("id"), - "repo_name": repo.get("name"), - "repo_display_name": repo.get("display_name"), - "provider_type": "gitlab", - "config": self.provider_id, - "contents": [{ - "content": runbook.get("content"), - "link": f"{self.gitlab_host}/api/v4/projects/{repo.get('id')}/repository/files/{runbook.get('file_path')}/raw", - "encoding": runbook.get("encoding"), - }], - "title": title, - } + def _format_content(self, runbookContent, repo): + """ + Format the content data into a dictionary. + """ + return { + "content": runbookContent.get("content"), + "link": f"{self.gitlab_host}/api/v4/projects/{repo.get('id')}/repository/files/{runbookContent.get('file_path')}/raw", + "encoding": runbook.get("encoding"), + "file_name": runbookContent.get("file_name"), + } + + + def _format_runbook(self, runbook, repo, title, md_path): + """ + Format the runbook data into a dictionary. + """ + if runbook is None: + raise Exception("Got empty runbook. Please check the runbook path and try again.") + + # Check if runbook is a list, if not convert to list + if isinstance(runbook, list): + runbook_contents = runbook + else: + runbook_contents = [runbook] + + # Filter runbook contents where type is "file" + filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents if runbookContent.get("type") == "file"] + + # Format the contents using a helper function + contents = [self._format_content(runbookContent, repo) for runbookContent in filtered_runbook_contents] + + # Return formatted runbook data as dictionary + return { + "relative_path": md_path, + "repo_id": repo.get("id"), + "repo_name": repo.get("name"), + "repo_display_name": repo.get("display_name"), + "provider_type": "gitlab", # This was changed from "github" to "gitlab", assuming it is intentional + "provider_id": self.provider_id, # Assuming this is supposed to be 'provider_id', not 'config' + "contents": contents, + "title": title, + } + def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): """Retrieve markdown files from the GitLab repository.""" @@ -272,7 +292,7 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): except HTTPError as e: raise Exception(f"Failed to get runbook: {e}") - return self._format_runbook(resp.json(), repo_meta, title) + return self._format_runbook(resp.json(), repo_meta, title, md_path) raise Exception("Failed to get runbook: repository or md_path not set") diff --git a/keep/runbooks/runbooks_service.py b/keep/runbooks/runbooks_service.py index 9473bb4a7..4b7a3ca3a 100644 --- a/keep/runbooks/runbooks_service.py +++ b/keep/runbooks/runbooks_service.py @@ -20,7 +20,7 @@ def create_runbook(session: Session, tenant_id: str, runbook_dto: dict): tenant_id=tenant_id, title=runbook_dto["title"], repo_id=runbook_dto["repo_id"], - relative_path=runbook_dto["file_path"], + relative_path=runbook_dto["relative_path"], provider_type=runbook_dto["provider_type"], provider_id=runbook_dto["provider_id"] ) @@ -50,11 +50,14 @@ def create_runbook(session: Session, tenant_id: str, runbook_dto: dict): logger.exception(f"Failed to create runbook {e}") @staticmethod - def get_all_runbooks(session: Session, tenant_id: str) -> List[RunbookDtoOut]: - runbooks = session.exec( - select(Runbook) - .where(Runbook.tenant_id == tenant_id) - .options(selectinload(Runbook.contents)).limit(1000) + def get_all_runbooks(session: Session, tenant_id: str, limit=25, offset=0) -> dict: + query = session.query(Runbook).filter( + Runbook.tenant_id == tenant_id, ) - - return [RunbookDtoOut.from_orm(runbook) for runbook in runbooks] \ No newline at end of file + + total_count = query.count() # Get the total count of runbooks matching the tenant_id + runbooks = query.options(selectinload(Runbook.contents)).limit(limit).offset(offset).all() # Fetch the paginated runbooks + result = [RunbookDtoOut.from_orm(runbook) for runbook in runbooks] # Convert runbooks to DTOs + + # Return total count and list of runbooks + return {"total_count": total_count, "runbooks": result} \ No newline at end of file From 1029b2ee558cdddd25e379d91ff8e4748a72fec9 Mon Sep 17 00:00:00 2001 From: Rajesh Jonnalagadda Date: Wed, 9 Oct 2024 21:21:54 +0530 Subject: [PATCH 2/3] chore: fix lint issues --- keep/providers/gitlab_provider/gitlab_provider.py | 2 +- keep/runbooks/runbooks_service.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/keep/providers/gitlab_provider/gitlab_provider.py b/keep/providers/gitlab_provider/gitlab_provider.py index 50c776818..78c0c65eb 100644 --- a/keep/providers/gitlab_provider/gitlab_provider.py +++ b/keep/providers/gitlab_provider/gitlab_provider.py @@ -235,7 +235,7 @@ def _format_content(self, runbookContent, repo): return { "content": runbookContent.get("content"), "link": f"{self.gitlab_host}/api/v4/projects/{repo.get('id')}/repository/files/{runbookContent.get('file_path')}/raw", - "encoding": runbook.get("encoding"), + "encoding": runbookContent.get("encoding"), "file_name": runbookContent.get("file_name"), } diff --git a/keep/runbooks/runbooks_service.py b/keep/runbooks/runbooks_service.py index 4b7a3ca3a..877144665 100644 --- a/keep/runbooks/runbooks_service.py +++ b/keep/runbooks/runbooks_service.py @@ -1,9 +1,8 @@ import logging -from typing import List from pydantic import ValidationError from sqlalchemy.orm import selectinload -from sqlmodel import Session, select +from sqlmodel import Session from keep.api.models.db.runbook import ( Runbook, RunbookContent, @@ -54,10 +53,10 @@ def get_all_runbooks(session: Session, tenant_id: str, limit=25, offset=0) -> di query = session.query(Runbook).filter( Runbook.tenant_id == tenant_id, ) - + total_count = query.count() # Get the total count of runbooks matching the tenant_id runbooks = query.options(selectinload(Runbook.contents)).limit(limit).offset(offset).all() # Fetch the paginated runbooks result = [RunbookDtoOut.from_orm(runbook) for runbook in runbooks] # Convert runbooks to DTOs - + # Return total count and list of runbooks return {"total_count": total_count, "runbooks": result} \ No newline at end of file From a49eaa02aaddf91e3b1d09466a12956e2328c744 Mon Sep 17 00:00:00 2001 From: Rajesh Jonnalagadda Date: Wed, 9 Oct 2024 22:12:15 +0530 Subject: [PATCH 3/3] fix: gitlab file content issue --- keep/providers/gitlab_provider/gitlab_provider.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/keep/providers/gitlab_provider/gitlab_provider.py b/keep/providers/gitlab_provider/gitlab_provider.py index 78c0c65eb..18a8951ae 100644 --- a/keep/providers/gitlab_provider/gitlab_provider.py +++ b/keep/providers/gitlab_provider/gitlab_provider.py @@ -254,7 +254,7 @@ def _format_runbook(self, runbook, repo, title, md_path): runbook_contents = [runbook] # Filter runbook contents where type is "file" - filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents if runbookContent.get("type") == "file"] + filtered_runbook_contents = [runbookContent for runbookContent in runbook_contents] # Format the contents using a helper function contents = [self._format_content(runbookContent, repo) for runbookContent in filtered_runbook_contents] @@ -279,9 +279,8 @@ def pull_runbook(self, repo=None, branch=None, md_path=None, title=None): md_path = md_path if md_path else self.authentication_config.md_path repo_meta = self.pull_repositories(project_id=repo) - - if repo_meta and branch and md_path: - repo_id = repo_meta.get("id") + repo_id = repo_meta.get("id") + if repo_id and branch and md_path: resp = requests.get( f"{self.gitlab_host}/api/v4/projects/{repo_id}/repository/files/{md_path}?ref={branch}", headers=self.__get_auth_header()