diff --git a/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx b/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx new file mode 100644 index 000000000000..26478514ca30 --- /dev/null +++ b/docs/core_docs/docs/integrations/document_loaders/web_loaders/jira.mdx @@ -0,0 +1,23 @@ +--- +sidebar_class_name: node-only +--- + +# Jira + +:::tip Compatibility +Only available on Node.js. +::: + +This covers how to load document objects from issues in a Jira projects. + +## Credentials + +- You'll need to set up an access token and provide it along with your jira username in order to authenticate the request +- You'll also need the `project key` for the project containing the issues to load as documents. + +## Usage + +import CodeBlock from "@theme/CodeBlock"; +import Example from "@examples/document_loaders/jira.ts"; + +{Example} diff --git a/examples/.env.example b/examples/.env.example index 2abb8d8e6912..9aae33991e92 100644 --- a/examples/.env.example +++ b/examples/.env.example @@ -83,4 +83,8 @@ FRIENDLI_TEAM=ADD_YOURS_HERE # https://suite.friendli.ai/ HANA_HOST=HANA_DB_ADDRESS HANA_PORT=HANA_DB_PORT HANA_UID=HANA_DB_USER -HANA_PWD=HANA_DB_PASSWORD \ No newline at end of file +HANA_PWD=HANA_DB_PASSWORD +JIRA_HOST=ADD_YOURS_HERE +JIRA_USERNAME=ADD_YOURS_HERE +JIRA_ACCESS_TOKEN=ADD_YOURS_HERE +JIRA_PROJECT_KEY=ADD_YOURS_HERE \ No newline at end of file diff --git a/examples/src/document_loaders/jira.ts b/examples/src/document_loaders/jira.ts new file mode 100644 index 000000000000..73d52efdf76f --- /dev/null +++ b/examples/src/document_loaders/jira.ts @@ -0,0 +1,26 @@ +import { JiraProjectLoader } from "@langchain/community/document_loaders/web/jira"; + +const host = process.env.JIRA_HOST || 'https://jira.example.com'; +const username = process.env.JIRA_USERNAME; +const accessToken = process.env.JIRA_ACCESS_TOKEN; +const projectKey = process.env.JIRA_PROJECT_KEY || 'PROJ'; + +if (username && accessToken) { + // Created within last 30 days + const createdAfter = new Date(); + createdAfter.setDate(createdAfter.getDate() - 30); + const loader = new JiraProjectLoader({ + host, + projectKey, + username, + accessToken, + createdAfter + }); + + const documents = await loader.load(); + console.log(`Loaded ${documents.length} Jira document(s)`); +} else { + console.log( + "You must provide a username and access token to run this example." + ); +} diff --git a/libs/langchain-community/.env.example b/libs/langchain-community/.env.example new file mode 100644 index 000000000000..2c36f95558b3 --- /dev/null +++ b/libs/langchain-community/.env.example @@ -0,0 +1,4 @@ +JIRA_HOST=ADD_YOURS_HERE +JIRA_USERNAME=ADD_YOURS_HERE +JIRA_ACCESS_TOKEN=ADD_YOURS_HERE +JIRA_PROJECT_KEY=ADD_YOURS_HERE diff --git a/libs/langchain-community/.gitignore b/libs/langchain-community/.gitignore index e6ae5fa54a4f..def745b41f80 100644 --- a/libs/langchain-community/.gitignore +++ b/libs/langchain-community/.gitignore @@ -902,6 +902,10 @@ document_loaders/web/imsdb.cjs document_loaders/web/imsdb.js document_loaders/web/imsdb.d.ts document_loaders/web/imsdb.d.cts +document_loaders/web/jira.cjs +document_loaders/web/jira.js +document_loaders/web/jira.d.ts +document_loaders/web/jira.d.cts document_loaders/web/figma.cjs document_loaders/web/figma.js document_loaders/web/figma.d.ts diff --git a/libs/langchain-community/langchain.config.js b/libs/langchain-community/langchain.config.js index 4a402c6941e8..7d735fa968ba 100644 --- a/libs/langchain-community/langchain.config.js +++ b/libs/langchain-community/langchain.config.js @@ -281,6 +281,7 @@ export const config = { "document_loaders/web/gitbook": "document_loaders/web/gitbook", "document_loaders/web/hn": "document_loaders/web/hn", "document_loaders/web/imsdb": "document_loaders/web/imsdb", + "document_loaders/web/jira": "document_loaders/web/jira", "document_loaders/web/figma": "document_loaders/web/figma", "document_loaders/web/firecrawl": "document_loaders/web/firecrawl", "document_loaders/web/github": "document_loaders/web/github", diff --git a/libs/langchain-community/package.json b/libs/langchain-community/package.json index 7b826ad1e106..423341bee8dc 100644 --- a/libs/langchain-community/package.json +++ b/libs/langchain-community/package.json @@ -68,7 +68,7 @@ "@cloudflare/workers-types": "^4.20230922.0", "@datastax/astra-db-ts": "^1.0.1", "@elastic/elasticsearch": "^8.4.0", - "@faker-js/faker": "^7.6.0", + "@faker-js/faker": "8.4.1", "@getmetal/metal-sdk": "^4.0.0", "@getzep/zep-cloud": "^1.0.6", "@getzep/zep-js": "^0.9.0", @@ -2746,6 +2746,15 @@ "import": "./document_loaders/web/imsdb.js", "require": "./document_loaders/web/imsdb.cjs" }, + "./document_loaders/web/jira": { + "types": { + "import": "./document_loaders/web/jira.d.ts", + "require": "./document_loaders/web/jira.d.cts", + "default": "./document_loaders/web/jira.d.ts" + }, + "import": "./document_loaders/web/jira.js", + "require": "./document_loaders/web/jira.cjs" + }, "./document_loaders/web/figma": { "types": { "import": "./document_loaders/web/figma.d.ts", @@ -4005,6 +4014,10 @@ "document_loaders/web/imsdb.js", "document_loaders/web/imsdb.d.ts", "document_loaders/web/imsdb.d.cts", + "document_loaders/web/jira.cjs", + "document_loaders/web/jira.js", + "document_loaders/web/jira.d.ts", + "document_loaders/web/jira.d.cts", "document_loaders/web/figma.cjs", "document_loaders/web/figma.js", "document_loaders/web/figma.d.ts", diff --git a/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts b/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts new file mode 100644 index 000000000000..e01d1d65663b --- /dev/null +++ b/libs/langchain-community/src/document_loaders/tests/jira.int.test.ts @@ -0,0 +1,209 @@ +/** + * NOTE: Env var should be set, and configured project should exist + */ +import { Document } from "@langchain/core/documents"; +import { expect, test } from "@jest/globals"; +import { + JiraIssue, + JiraProjectLoader, + JiraProjectLoaderParams, +} from "../web/jira.js"; + +describe("JiraProjectLoader Integration Tests", () => { + const JIRA_HOST = requireEnvVar("JIRA_HOST"); + const JIRA_USERNAME = requireEnvVar("JIRA_USERNAME"); + const JIRA_ACCESS_TOKEN = requireEnvVar("JIRA_ACCESS_TOKEN"); + const JIRA_PROJECT_KEY = requireEnvVar("JIRA_PROJECT_KEY"); + const jiraConf: JiraProjectLoaderParams = { + host: JIRA_HOST, + projectKey: JIRA_PROJECT_KEY, + username: JIRA_USERNAME, + accessToken: JIRA_ACCESS_TOKEN, + limitPerRequest: 20, + }; + + test("should load Jira project issues as documents successfully", async () => { + const docs = await loadJiraDocsUntil((docs) => docs.length > 0); + + expect(docs).toBeDefined(); + expect(Array.isArray(docs)).toBe(true); + + if (docs.length < 1) { + // Skip test if not enough issues available + return; + } + const firstDoc = docs[0]; + + // Check document structure + expect(firstDoc).toHaveProperty("pageContent"); + expect(firstDoc).toHaveProperty("metadata"); + + // Check metadata + expect(firstDoc.metadata).toHaveProperty("id"); + expect(firstDoc.metadata).toHaveProperty("host", JIRA_HOST); + expect(firstDoc.metadata).toHaveProperty("projectKey", JIRA_PROJECT_KEY); + + // Check pageContent contains essential Jira issue information + const content = firstDoc.pageContent; + expect(content).toContain("Issue:"); + expect(content).toContain("Project:"); + expect(content).toContain("Status:"); + expect(content).toContain("Priority:"); + expect(content).toContain("Type:"); + expect(content).toContain("Creator:"); + }); + + test("should filter issues based on createdAfter date", async () => { + // First load at least 2 issues with different creation dates (ignoring time) + const baseIssues = await loadJiraIssuesUntil(haveTwoDifferentCreationDates); + if (baseIssues.length < 2) { + // Skip test if not enough issues available + return; + } + + // Create a map from date string without time to list of issues + const dateToIssueMap = new Map(); + baseIssues.forEach((issue) => { + const date = asStringWithoutTime(new Date(issue.fields.created)); + dateToIssueMap.set(date, (dateToIssueMap.get(date) ?? []).concat(issue)); + }); + // Convert map to list of {date, issues} + const issuesGroupedByDate = Array.from( + dateToIssueMap, + ([date, issues]) => ({ date, issues }) + ); + issuesGroupedByDate.sort((a, b) => a.date.localeCompare(b.date)); + + // Pick middle date to split issues in two groups + const middleIndex = Math.floor(issuesGroupedByDate.length / 2); + const middleDate = new Date(issuesGroupedByDate[middleIndex].date); + const issuesAfterMiddle = issuesGroupedByDate + .slice(middleIndex) + .flatMap(({ issues }) => issues); + + // Load issues created after middle date + const loader = new JiraProjectLoader({ + ...jiraConf, + createdAfter: middleDate, + }); + + const filteredDocs = await loader.load(); + + // Verify we got the expected issues + expect(filteredDocs.length).toBeGreaterThan(0); + expect(filteredDocs.length).toBeLessThan(baseIssues.length); + + // Verify all returned issues are created after our cutoff date + const middleDateTimestamp = middleDate.getTime(); + filteredDocs.forEach((doc) => { + const issueDateString = doc.pageContent + .split("\n") + .filter((line) => /^Created: /.test(line))[0] + .replace("Created: ", ""); + const issueDateTimestamp = new Date( + asStringWithoutTime(new Date(issueDateString)) + ).getTime(); + expect(issueDateTimestamp).toBeGreaterThanOrEqual(middleDateTimestamp); + }); + + // Verify we got the same issues as in our original set + const filteredIds = new Set(filteredDocs.map((d) => d.metadata.id)); + const expectedIds = new Set(issuesAfterMiddle.map((issue) => issue.id)); + expect(filteredIds).toEqual(expectedIds); + }); + + test("should handle invalid credentials", async () => { + const loader = new JiraProjectLoader({ + ...jiraConf, + username: "invalid_username", + accessToken: "invalid_token", + }); + + const docs = await loader.load(); + expect(docs).toEqual([]); + }); + + test("should handle invalid project key", async () => { + const loader = new JiraProjectLoader({ + ...jiraConf, + projectKey: "INVALID_PROJECT_KEY", + }); + + const docs = await loader.load(); + expect(docs).toEqual([]); + }); + + function requireEnvVar(name: string): string { + // eslint-disable-next-line no-process-env + const value = process.env[name]; + if (!value) { + throw new Error(`environment variable "${name}" must be set`); + } + return value; + } + + function asStringWithoutTime(date: Date): string { + return date.toISOString().split("T")[0]; + } + + function sameDate(a: Date, b: Date) { + return asStringWithoutTime(a) === asStringWithoutTime(b); + } + + function haveTwoDifferentCreationDates(issues: JiraIssue[]): boolean { + return ( + issues.length >= 2 && + issues + .slice(1) + .some( + (issue) => + !sameDate( + new Date(issue.fields.created), + new Date(issues[0].fields.created) + ) + ) + ); + } + + async function loadJiraDocsUntil(predicate: (docs: Document[]) => boolean) { + const load = (createdAfter: Date) => + new JiraProjectLoader({ + ...jiraConf, + createdAfter, + }).load(); + return loadUntil(load, predicate); + } + + async function loadJiraIssuesUntil( + predicate: (docs: JiraIssue[]) => boolean + ) { + const load = (createdAfter: Date) => + new JiraProjectLoader({ + ...jiraConf, + createdAfter, + }).loadAsIssues(); + return loadUntil(load, predicate); + } + + async function loadUntil( + loadCreatedAfter: (date: Date) => Promise, + predicate: (loaded: T[]) => boolean + ): Promise { + const now = new Date(); + let months = 1; + const maxMonths = 120; + + let loaded: T[] = []; + while (!predicate(loaded) && months < maxMonths) { + const createdAfter = new Date(now); + createdAfter.setDate(now.getDate() - months * 30); + loaded = await loadCreatedAfter(createdAfter); + months *= 1.2; + } + + if (months >= maxMonths) { + return []; + } + return loaded; + } +}); diff --git a/libs/langchain-community/src/document_loaders/tests/jira.test.ts b/libs/langchain-community/src/document_loaders/tests/jira.test.ts new file mode 100644 index 000000000000..92b1224446e0 --- /dev/null +++ b/libs/langchain-community/src/document_loaders/tests/jira.test.ts @@ -0,0 +1,267 @@ +import { faker } from "@faker-js/faker"; +import { + JiraDocumentConverter, + JiraIssue, + JiraUser, + JiraIssueType, + JiraPriority, + JiraProgress, + JiraProject, + JiraStatus, + JiraStatusCategory, +} from "../web/jira.js"; + +describe("JiraDocumentConverter Unit Tests", () => { + function getConverter() { + return new JiraDocumentConverter({ + projectKey: "PROJ", + host: "https://example.com", + }); + } + + it("should handle missing optional fields", () => { + const issue: JiraIssue = someJiraIssue(); + delete issue.fields.assignee; + delete issue.fields.duedate; + + const converter = getConverter(); + const document = converter.convertToDocuments([issue])[0]; + + expect(document).toBeDefined(); + expect(document.pageContent).toContain(issue.fields.summary); + expect(document.pageContent).toContain("Assignee: Unassigned"); + expect(document.pageContent).not.toMatch(/.*^Due Date: .*/m); + expect(document.metadata).toEqual({ + id: issue.id, + host: converter.host, + projectKey: converter.projectKey, + }); + }); + + it("should format the document content properly", () => { + const converter = getConverter(); + const issue = someJiraIssue(); + const document = converter.convertToDocuments([issue])[0]; + + expect(document.pageContent).toContain(issue.fields.summary); + expect(document.pageContent).toContain(issue.fields.description); + expect(document.pageContent).toContain( + issue.fields.labels?.join(", ") || "" + ); + expect(document.pageContent).toContain( + issue.fields.reporter?.displayName || "" + ); + expect(document.pageContent).toContain( + issue.fields.assignee?.displayName || "Unassigned" + ); + expect(document.pageContent).toContain(issue.fields.duedate || ""); + expect(document.pageContent).toContain( + issue.fields.timeestimate?.toString() || "" + ); + expect(document.pageContent).toContain( + issue.fields.timespent?.toString() || "" + ); + expect(document.pageContent).toContain(issue.fields.resolutiondate || ""); + expect(document.pageContent).toContain( + issue.fields.progress.percent?.toString() || "" + ); + }); +}); + +export function someJiraIssueType( + overrides: Partial = {} +): JiraIssueType { + const baseIssueType: JiraIssueType = { + avatarId: faker.number.int({ min: 1, max: 100 }), + description: faker.lorem.sentence(), + entityId: faker.string.uuid(), + hierarchyLevel: faker.number.int({ min: 1, max: 5 }), + iconUrl: faker.image.url(), + id: faker.string.numeric(5), + name: faker.helpers.arrayElement(["Bug", "Task", "Story", "Epic"]), + self: faker.internet.url(), + subtask: false, + }; + + return { + ...baseIssueType, + ...overrides, + }; +} + +export function someJiraUser(overrides: Partial = {}): JiraUser { + const baseUser = { + accountId: faker.string.uuid(), + accountType: "atlassian", + active: true, + avatarUrls: { + "16x16": faker.image.avatar(), + "24x24": faker.image.avatar(), + "32x32": faker.image.avatar(), + "48x48": faker.image.avatar(), + }, + displayName: faker.person.fullName(), + emailAddress: faker.internet.email(), + self: faker.internet.url(), + timeZone: faker.location.timeZone(), + }; + + return { + ...baseUser, + ...overrides, + }; +} + +export function someJiraPriority( + overrides: Partial = {} +): JiraPriority { + const basePriority: JiraPriority = { + iconUrl: faker.image.url(), + id: faker.string.numeric(2), + name: faker.helpers.arrayElement([ + "Highest", + "High", + "Medium", + "Low", + "Lowest", + ]), + self: faker.internet.url(), + }; + + return { + ...basePriority, + ...overrides, + }; +} + +export function someJiraProgress( + overrides: Partial = {} +): JiraProgress { + const baseProgress: JiraProgress = { + progress: faker.number.int({ min: 0, max: 100 }), + total: 100, + percent: faker.number.int({ min: 0, max: 100 }), + }; + + return { + ...baseProgress, + ...overrides, + }; +} + +export function someJiraProject( + overrides: Partial = {} +): JiraProject { + const baseProject: JiraProject = { + avatarUrls: { + "16x16": faker.image.avatar(), + "24x24": faker.image.avatar(), + "32x32": faker.image.avatar(), + "48x48": faker.image.avatar(), + }, + id: faker.string.numeric(5), + key: faker.string.alpha(4).toUpperCase(), + name: faker.company.name(), + projectTypeKey: "software", + self: faker.internet.url(), + simplified: false, + }; + + return { + ...baseProject, + ...overrides, + }; +} + +export function someJiraStatusCategory( + overrides: Partial = {} +): JiraStatusCategory { + const baseStatusCategory: JiraStatusCategory = { + self: faker.internet.url(), + id: faker.number.int({ min: 1, max: 5 }), + key: faker.helpers.arrayElement(["new", "indeterminate", "done"]), + colorName: faker.helpers.arrayElement(["blue-gray", "yellow", "green"]), + name: faker.helpers.arrayElement(["To Do", "In Progress", "Done"]), + }; + + return { + ...baseStatusCategory, + ...overrides, + }; +} + +export function someJiraStatus( + overrides: Partial = {} +): JiraStatus { + const baseStatus: JiraStatus = { + self: faker.internet.url(), + description: faker.lorem.sentence(), + iconUrl: faker.image.url(), + name: faker.helpers.arrayElement([ + "To Do", + "In Progress", + "Done", + "Blocked", + ]), + id: faker.string.numeric(2), + statusCategory: someJiraStatusCategory(), + }; + + return { + ...baseStatus, + ...overrides, + }; +} + +export function someJiraIssue(overrides: Partial = {}): JiraIssue { + const issueKey = `${faker.string.alpha(4).toUpperCase()}-${faker.number.int({ + min: 1, + max: 9999, + })}`; + + const baseIssue: JiraIssue = { + expand: "renderedFields", + id: faker.string.numeric(5), + self: `https://${faker.internet.domainName()}/rest/api/2/issue/${issueKey}`, + key: issueKey, + fields: { + assignee: faker.datatype.boolean() ? someJiraUser() : undefined, + created: faker.date.past().toISOString(), + description: faker.lorem.paragraph(), + issuelinks: [], + issuetype: someJiraIssueType(), + labels: faker.datatype.boolean() + ? Array.from({ length: faker.number.int({ min: 1, max: 5 }) }, () => + faker.word.noun() + ) + : undefined, + priority: someJiraPriority(), + progress: someJiraProgress(), + project: someJiraProject(), + reporter: faker.datatype.boolean() ? someJiraUser() : undefined, + creator: someJiraUser(), + resolutiondate: faker.datatype.boolean() + ? faker.date.recent().toISOString() + : undefined, + status: someJiraStatus(), + subtasks: [], + summary: faker.lorem.sentence(), + timeestimate: faker.datatype.boolean() + ? faker.number.int({ min: 1, max: 100 }) * 3600 + : undefined, + timespent: faker.datatype.boolean() + ? faker.number.int({ min: 1, max: 100 }) * 3600 + : undefined, + updated: faker.date.recent().toISOString(), + duedate: faker.datatype.boolean() + ? faker.date.future().toISOString() + : undefined, + }, + }; + console.log(baseIssue.fields.duedate); + + return { + ...baseIssue, + ...overrides, + }; +} diff --git a/libs/langchain-community/src/document_loaders/web/jira.ts b/libs/langchain-community/src/document_loaders/web/jira.ts new file mode 100644 index 000000000000..59e0879d2ab9 --- /dev/null +++ b/libs/langchain-community/src/document_loaders/web/jira.ts @@ -0,0 +1,441 @@ +import { Document } from "@langchain/core/documents"; +import { BaseDocumentLoader } from "@langchain/core/document_loaders/base"; + +export type JiraStatusCategory = { + self: string; + id: number; + key: string; + colorName: string; + name: string; +}; + +export type JiraStatus = { + self: string; + description: string; + iconUrl: string; + name: string; + id: string; + statusCategory: JiraStatusCategory; +}; + +export type JiraUser = { + accountId: string; + accountType: string; + active: boolean; + avatarUrls: { + "16x16": string; + "24x24": string; + "32x32": string; + "48x48": string; + }; + displayName: string; + emailAddress: string; + self: string; + timeZone: string; +}; + +export type JiraIssueType = { + avatarId: number; + description: string; + entityId: string; + hierarchyLevel: number; + iconUrl: string; + id: string; + name: string; + self: string; + subtask: boolean; +}; + +export type JiraPriority = { + iconUrl: string; + id: string; + name: string; + self: string; +}; + +export type JiraProgress = { + progress: number; + total: number; + percent?: number; +}; + +export type JiraProject = { + avatarUrls: { + "16x16": string; + "24x24": string; + "32x32": string; + "48x48": string; + }; + id: string; + key: string; + name: string; + projectTypeKey: string; + self: string; + simplified: boolean; +}; + +export type JiraSubTask = { + id: string; + key: string; + self: string; + fields: { + issuetype: JiraIssueType; + priority: JiraPriority; + status: JiraStatus; + summary: string; + }; +}; + +export type JiraIssueLinkType = { + id: string; + name: string; + inward: string; + outward: string; + self: string; +}; + +export type JiraBriefIssue = { + id: string; + key: string; + self: string; + fields: { + summary: string; + status: JiraStatus; + priority: JiraPriority; + issuetype: JiraIssueType; + }; +}; + +export type JiraIssueLink = { + id: string; + self: string; + type: JiraIssueLinkType; + inwardIssue?: JiraBriefIssue; + outwardIssue?: JiraBriefIssue; +}; + +export type JiraIssue = { + expand: string; + id: string; + self: string; + key: string; + fields: { + assignee?: JiraUser; + created: string; + description: string; + issuelinks: JiraIssueLink[]; + issuetype: JiraIssueType; + labels?: string[]; + priority: JiraPriority; + progress: JiraProgress; + project: JiraProject; + reporter?: JiraUser; + creator: JiraUser; + resolutiondate?: string; + status: JiraStatus; + subtasks: JiraSubTask[]; + summary: string; + timeestimate?: number; + timespent?: number; + updated: string; + duedate?: string; + parent?: JiraBriefIssue; + }; +}; + +export type JiraAPIResponse = { + expand: string; + startAt: number; + maxResults: number; + total: number; + issues: JiraIssue[]; +}; + +/** + * Interface representing the parameters for configuring the + * JiraDocumentConverter. + */ +export interface JiraDocumentConverterParams { + host: string; + projectKey: string; +} + +/** + * Class responsible for converting Jira issues to Document objects + */ +export class JiraDocumentConverter { + public readonly host: string; + + public readonly projectKey: string; + + constructor({ host, projectKey }: JiraDocumentConverterParams) { + this.host = host; + this.projectKey = projectKey; + } + + public convertToDocuments(issues: JiraIssue[]): Document[] { + return issues.map((issue) => this.documentFromIssue(issue)); + } + + private documentFromIssue(issue: JiraIssue): Document { + return new Document({ + pageContent: this.formatIssueInfo({ + issue, + host: this.host, + }), + metadata: { + id: issue.id, + host: this.host, + projectKey: this.projectKey, + }, + }); + } + + private formatIssueInfo({ + issue, + host, + }: { + issue: JiraIssue; + host: string; + }): string { + let text = `Issue: ${this.formatMainIssueInfoText({ issue, host })}\n`; + text += `Project: ${issue.fields.project.name} (${issue.fields.project.key}, ID ${issue.fields.project.id})\n`; + text += `Status: ${issue.fields.status.name}\n`; + text += `Priority: ${issue.fields.priority.name}\n`; + text += `Type: ${issue.fields.issuetype.name}\n`; + text += `Creator: ${issue.fields.creator.displayName}\n`; + + if (issue.fields.labels && issue.fields.labels.length > 0) { + text += `Labels: ${issue.fields.labels.join(", ")}\n`; + } + + text += `Created: ${issue.fields.created}\n`; + text += `Updated: ${issue.fields.updated}\n`; + + if (issue.fields.reporter) { + text += `Reporter: ${issue.fields.reporter.displayName}\n`; + } + + text += `Assignee: ${issue.fields.assignee?.displayName ?? "Unassigned"}\n`; + + if (issue.fields.duedate) { + text += `Due Date: ${issue.fields.duedate}\n`; + } + + if (issue.fields.timeestimate) { + text += `Time Estimate: ${issue.fields.timeestimate}\n`; + } + + if (issue.fields.timespent) { + text += `Time Spent: ${issue.fields.timespent}\n`; + } + + if (issue.fields.resolutiondate) { + text += `Resolution Date: ${issue.fields.resolutiondate}\n`; + } + + if (issue.fields.description) { + text += `Description: ${issue.fields.description}\n`; + } + + if (issue.fields.progress.percent) { + text += `Progress: ${issue.fields.progress.percent}%\n`; + } + + if (issue.fields.parent) { + text += `Parent Issue: ${this.formatMainIssueInfoText({ + issue: issue.fields.parent, + host, + })}\n`; + } + + if (issue.fields.subtasks.length > 0) { + text += `Subtasks:\n`; + issue.fields.subtasks.forEach((subtask) => { + text += ` - ${this.formatMainIssueInfoText({ + issue: subtask, + host, + })}\n`; + }); + } + + if (issue.fields.issuelinks.length > 0) { + text += `Issue Links:\n`; + issue.fields.issuelinks.forEach((link) => { + text += ` - ${link.type.name}\n`; + if (link.inwardIssue) { + text += ` - ${this.formatMainIssueInfoText({ + issue: link.inwardIssue, + host, + })}\n`; + } + if (link.outwardIssue) { + text += ` - ${this.formatMainIssueInfoText({ + issue: link.outwardIssue, + host, + })}\n`; + } + }); + } + + return text; + } + + private getLinkToIssue({ + issueKey, + host, + }: { + issueKey: string; + host: string; + }): string { + return `${host}/browse/${issueKey}`; + } + + private formatMainIssueInfoText({ + issue, + host, + }: { + issue: JiraIssue | JiraBriefIssue; + host: string; + }): string { + const link = this.getLinkToIssue({ + issueKey: issue.key, + host, + }); + + const text = `${issue.key} (ID ${issue.id}) - ${issue.fields.summary} (${link})`; + + return text; + } +} + +/** + * Interface representing the parameters for configuring the + * JiraProjectLoader. + */ +export interface JiraProjectLoaderParams { + host: string; + projectKey: string; + username: string; + accessToken: string; + limitPerRequest?: number; + createdAfter?: Date; +} + +const API_ENDPOINTS = { + SEARCH: "/rest/api/2/search", +}; + +/** + * Class representing a document loader for loading pages from Confluence. + */ +export class JiraProjectLoader extends BaseDocumentLoader { + private readonly accessToken: string; + + public readonly host: string; + + public readonly projectKey: string; + + public readonly username: string; + + public readonly limitPerRequest: number; + + private readonly createdAfter?: Date; + + private readonly documentConverter: JiraDocumentConverter; + + constructor({ + host, + projectKey, + username, + accessToken, + limitPerRequest = 100, + createdAfter, + }: JiraProjectLoaderParams) { + super(); + this.host = host; + this.projectKey = projectKey; + this.username = username; + this.accessToken = accessToken; + this.limitPerRequest = limitPerRequest; + this.createdAfter = createdAfter; + this.documentConverter = new JiraDocumentConverter({ host, projectKey }); + } + + private buildAuthorizationHeader(): string { + return `Basic ${Buffer.from( + `${this.username}:${this.accessToken}` + ).toString("base64")}`; + } + + public async load(): Promise { + try { + const allJiraIssues = await this.loadAsIssues(); + return this.documentConverter.convertToDocuments(allJiraIssues); + } catch (error) { + console.error("Error:", error); + return []; + } + } + + public async loadAsIssues(): Promise { + const allIssues: JiraIssue[] = []; + + for await (const issues of this.fetchIssues()) { + allIssues.push(...issues); + } + + return allIssues; + } + + protected toJiraDateString(date: Date | undefined): string | undefined { + if (!date) { + return undefined; + } + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, "0"); + const dayOfMonth = String(date.getDate()).padStart(2, "0"); + return `${year}-${month}-${dayOfMonth}`; + } + + protected async *fetchIssues(): AsyncIterable { + const authorizationHeader = this.buildAuthorizationHeader(); + const url = `${this.host}${API_ENDPOINTS.SEARCH}`; + const createdAfterAsString = this.toJiraDateString(this.createdAfter); + let startAt = 0; + + while (true) { + try { + const jqlProps = [ + `project=${this.projectKey}`, + ...(createdAfterAsString ? [`created>=${createdAfterAsString}`] : []), + ]; + const params = new URLSearchParams({ + jql: jqlProps.join(" AND "), + startAt: `${startAt}`, + maxResults: `${this.limitPerRequest}`, + }); + const pageUrl = `${url}?${params}`; + + const options = { + method: "GET", + headers: { + Authorization: authorizationHeader, + Accept: "application/json", + }, + }; + + const response = await fetch(pageUrl, options); + const data: JiraAPIResponse = await response.json(); + + if (!data.issues || data.issues.length === 0) break; + + yield data.issues; + startAt += this.limitPerRequest; + } catch (error) { + console.error(error); + yield []; + } + } + } +} diff --git a/libs/langchain-community/src/load/import_map.ts b/libs/langchain-community/src/load/import_map.ts index 8b3b734a82c1..26aa80ca068b 100644 --- a/libs/langchain-community/src/load/import_map.ts +++ b/libs/langchain-community/src/load/import_map.ts @@ -73,6 +73,7 @@ export * as indexes__base from "../indexes/base.js"; export * as indexes__memory from "../indexes/memory.js"; export * as document_loaders__web__airtable from "../document_loaders/web/airtable.js"; export * as document_loaders__web__html from "../document_loaders/web/html.js"; +export * as document_loaders__web__jira from "../document_loaders/web/jira.js"; export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js"; export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js"; export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js"; diff --git a/yarn.lock b/yarn.lock index 5e6b6ed60a57..6aa947124f55 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10048,6 +10048,13 @@ __metadata: languageName: node linkType: hard +"@faker-js/faker@npm:8.4.1, @faker-js/faker@npm:^8.4.1": + version: 8.4.1 + resolution: "@faker-js/faker@npm:8.4.1" + checksum: d802d531f8929562715adc279cfec763c9a4bc596ec67b0ce43fd0ae61b285d2b0eec6f1f4aa852452a63721a842fe7e81926dce7bd92acca94b01e2a1f55f5a + languageName: node + linkType: hard + "@faker-js/faker@npm:^7.6.0": version: 7.6.0 resolution: "@faker-js/faker@npm:7.6.0" @@ -10069,13 +10076,6 @@ __metadata: languageName: node linkType: hard -"@faker-js/faker@npm:^8.4.1": - version: 8.4.1 - resolution: "@faker-js/faker@npm:8.4.1" - checksum: d802d531f8929562715adc279cfec763c9a4bc596ec67b0ce43fd0ae61b285d2b0eec6f1f4aa852452a63721a842fe7e81926dce7bd92acca94b01e2a1f55f5a - languageName: node - linkType: hard - "@fastify/busboy@npm:^1.2.1": version: 1.2.1 resolution: "@fastify/busboy@npm:1.2.1" @@ -11709,7 +11709,7 @@ __metadata: "@cloudflare/workers-types": ^4.20230922.0 "@datastax/astra-db-ts": ^1.0.1 "@elastic/elasticsearch": ^8.4.0 - "@faker-js/faker": ^7.6.0 + "@faker-js/faker": 8.4.1 "@getmetal/metal-sdk": ^4.0.0 "@getzep/zep-cloud": ^1.0.6 "@getzep/zep-js": ^0.9.0