Quest Changes for Custom Partition

added test cases for - time partition - column partition - time and column partition
parseablehq · May 24, 2024 · 6fe04df · 6fe04df
1 parent 44a13d6
commit 6fe04df
Show file tree

Hide file tree

Showing 3 changed files with 122 additions and 2 deletions.
diff --git a/quest_test.go b/quest_test.go
@@ -261,6 +261,102 @@ func TestSmokeLoadWithK6Stream(t *testing.T) {
 	DeleteStream(t, NewGlob.QueryClient, NewGlob.Stream)
 }
 
+func TestSmokeLoad_TimePartition_WithK6Stream(t *testing.T) {
+	time_partition_stream := NewGlob.Stream + "timepartition"
+	timeHeader := map[string]string{"X-P-Time-Partition": "source_time", "X-P-Time-Partition-Limit": "365d"}
+	CreateStreamWithHeader(t, NewGlob.QueryClient, time_partition_stream, timeHeader)
+	if NewGlob.IngestorUrl.String() == "" {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.QueryUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.QueryUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.QueryPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", time_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	} else {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.IngestorUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.IngestorUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.IngestorPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", time_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	}
+	time.Sleep(60 * time.Second)
+	QueryLogStreamCount_WithTimePartition(t, NewGlob.QueryClient, time_partition_stream, 60000)
+	DeleteStream(t, NewGlob.QueryClient, time_partition_stream)
+}
+
+func TestSmokeLoad_CustomPartition_WithK6Stream(t *testing.T) {
+	custom_partition_stream := NewGlob.Stream + "custompartition"
+	customHeader := map[string]string{"X-P-Custom-Partition": "level,os"}
+	CreateStreamWithHeader(t, NewGlob.QueryClient, custom_partition_stream, customHeader)
+	if NewGlob.IngestorUrl.String() == "" {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.QueryUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.QueryUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.QueryPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", custom_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	} else {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.IngestorUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.IngestorUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.IngestorPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", custom_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	}
+	time.Sleep(60 * time.Second)
+	QueryLogStreamCount(t, NewGlob.QueryClient, custom_partition_stream, 60000)
+	DeleteStream(t, NewGlob.QueryClient, custom_partition_stream)
+}
+
+func TestSmokeLoad_TimeAndCustomPartition_WithK6Stream(t *testing.T) {
+	custom_partition_stream := NewGlob.Stream + "timecustompartition"
+	customHeader := map[string]string{"X-P-Custom-Partition": "level,os", "X-P-Time-Partition": "source_time", "X-P-Time-Partition-Limit": "365d"}
+	CreateStreamWithHeader(t, NewGlob.QueryClient, custom_partition_stream, customHeader)
+	if NewGlob.IngestorUrl.String() == "" {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.QueryUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.QueryUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.QueryPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", custom_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	} else {
+		cmd := exec.Command("k6",
+			"run",
+			"-e", fmt.Sprintf("P_URL=%s", NewGlob.IngestorUrl.String()),
+			"-e", fmt.Sprintf("P_USERNAME=%s", NewGlob.IngestorUsername),
+			"-e", fmt.Sprintf("P_PASSWORD=%s", NewGlob.IngestorPassword),
+			"-e", fmt.Sprintf("P_STREAM=%s", custom_partition_stream),
+			"./scripts/smoke.js")
+
+		cmd.Run()
+		cmd.Output()
+	}
+	time.Sleep(60 * time.Second)
+	QueryLogStreamCount_WithTimePartition(t, NewGlob.QueryClient, custom_partition_stream, 60000)
+	DeleteStream(t, NewGlob.QueryClient, custom_partition_stream)
+}
+
 func TestSmokeSetAlert(t *testing.T) {
 	CreateStream(t, NewGlob.QueryClient, NewGlob.Stream)
 	if NewGlob.IngestorUrl.String() == "" {

diff --git a/scripts/smoke.js b/scripts/smoke.js
@@ -16,8 +16,12 @@ export const options = {
 };
 
 function current_time() {
-    let event = new Date();
-    return event.toISOString();
+    const startDate = new Date('2024-05-17T15:00:00');
+    const endDate = new Date('2024-05-17T15:03:59');
+    const timeDiff = endDate.getTime() - startDate.getTime();
+    const randomTime = Math.random() * timeDiff;
+    const randomDate = new Date(startDate.getTime() + randomTime);
+    return(randomDate.toISOString());
 }
 
 function schemas() {

diff --git a/test_utils.go b/test_utils.go
@@ -265,6 +265,26 @@ func QueryLogStreamCount(t *testing.T, client HTTPClient, stream string, count u
 	require.Equalf(t, expected, body, "Query count incorrect; Expected %s, Actual %s", expected, body)
 }
 
+func QueryLogStreamCount_WithTimePartition(t *testing.T, client HTTPClient, stream string, count uint64) {
+	// Query last 10 minutes of data only
+	endTime := "2024-05-17T10:00:00.000Z"
+	startTime := "2024-05-17T08:00:00.000Z"
+
+	query := map[string]interface{}{
+		"query":     "select count(*) as count from " + stream,
+		"startTime": startTime,
+		"endTime":   endTime,
+	}
+	queryJSON, _ := json.Marshal(query)
+	req, _ := client.NewRequest("POST", "query", bytes.NewBuffer(queryJSON))
+	response, err := client.Do(req)
+	require.NoErrorf(t, err, "Request failed: %s", err)
+	body := readAsString(response.Body)
+	require.Equalf(t, 200, response.StatusCode, "Server returned http code: %s and response: %s", response.Status, body)
+	expected := fmt.Sprintf(`[{"count":%d}]`, count)
+	require.Equalf(t, expected, body, "Query count incorrect; Expected %s, Actual %s", expected, body)
+}
+
 func QueryTwoLogStreamCount(t *testing.T, client HTTPClient, stream1 string, stream2 string, count uint64) {
 	// Query last 10 minutes of data only
 	endTime := time.Now().Add(time.Second).Format(time.RFC3339Nano)