From 68677d8eb3366414ba059dc0d35f670330d91174 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 07:26:34 +0000 Subject: [PATCH 1/8] Return 409 error when canceling completed crawl/batch scrape jobs - Add completion check to v0, v1, and v2 crawl cancel controllers - Return 409 Conflict with clear error message for completed jobs - Update OpenAPI specifications to document new 409 response - Add comprehensive tests for both completed and in-progress job cancellation - Maintain existing functionality for canceling in-progress jobs Co-Authored-By: Micah Stairs --- apps/api/openapi.json | 32 ++++++++ .../__tests__/snips/v1/batch-scrape.test.ts | 82 +++++++++++++++++++ apps/api/src/__tests__/snips/v1/crawl.test.ts | 72 ++++++++++++++++ apps/api/src/controllers/v0/crawl-cancel.ts | 29 ++++++- apps/api/src/controllers/v1/crawl-cancel.ts | 30 ++++++- apps/api/src/controllers/v2/crawl-cancel.ts | 30 ++++++- apps/api/v1-openapi.json | 32 ++++++++ 7 files changed, 304 insertions(+), 3 deletions(-) diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 716a94901a..59aa3bb921 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -378,6 +378,22 @@ } } }, + "409": { + "description": "Job already completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Cannot cancel job that has already completed" + } + } + } + } + } + }, "500": { "description": "Server error", "content": { @@ -607,6 +623,22 @@ } } }, + "409": { + "description": "Job already completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Cannot cancel job that has already completed" + } + } + } + } + } + }, "500": { "description": "Server error", "content": { diff --git a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts index 23dd503ebe..65e18c3d43 100644 --- a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts +++ b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts @@ -94,4 +94,86 @@ describe("Batch scrape tests", () => { }, scrapeTimeout, ); + + describe("DELETE /v1/batch/scrape/{id}", () => { + it.concurrent( + "should return 409 when trying to cancel a completed batch scrape job", + async () => { + const response = await batchScrape( + { + urls: ["https://example.com"], + }, + identity, + ); + + expect(response.success).toBe(true); + if (response.success) { + const batchId = response.id; + + const cancelResponse = await fetch( + `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${identity.apiKey}`, + "Content-Type": "application/json", + }, + }, + ); + + expect(cancelResponse.status).toBe(409); + const cancelData = await cancelResponse.json(); + expect(cancelData.error).toBe( + "Cannot cancel job that has already completed", + ); + } + }, + scrapeTimeout, + ); + + it.concurrent( + "should successfully cancel an in-progress batch scrape job", + async () => { + const startResponse = await fetch( + `${process.env.TEST_API_URL}/v1/batch/scrape`, + { + method: "POST", + headers: { + Authorization: `Bearer ${identity.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + urls: Array.from( + { length: 10 }, + (_, i) => `https://firecrawl.dev/page-${i}`, + ), + }), + }, + ); + + expect(startResponse.status).toBe(200); + const startData = await startResponse.json(); + const batchId = startData.id; + + await new Promise(resolve => setTimeout(resolve, 1000)); + + const cancelResponse = await fetch( + `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${identity.apiKey}`, + "Content-Type": "application/json", + }, + }, + ); + + expect(cancelResponse.status).toBe(200); + const cancelData = await cancelResponse.json(); + expect(cancelData.success).toBe(true); + expect(cancelData.message).toContain("cancelled"); + }, + scrapeTimeout, + ); + }); }); diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts index ce0550c66e..ede23c27a7 100644 --- a/apps/api/src/__tests__/snips/v1/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts @@ -386,4 +386,76 @@ describe("Crawl tests", () => { expect(typeof response.body.id).toBe("string"); }, ); + + describe("POST /v1/crawl/{id}/cancel", () => { + it.concurrent( + "should return 409 when trying to cancel a completed crawl job", + async () => { + const res = await crawl( + { + url: "https://example.com", + limit: 1, + }, + identity, + ); + + expect(res.success).toBe(true); + if (res.success) { + const crawlId = res.id; + + const cancelResponse = await fetch( + `${process.env.TEST_API_URL}/v1/crawl/${crawlId}/cancel`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${identity.apiKey}`, + "Content-Type": "application/json", + }, + }, + ); + + expect(cancelResponse.status).toBe(409); + const cancelData = await cancelResponse.json(); + expect(cancelData.error).toBe( + "Cannot cancel job that has already completed", + ); + } + }, + scrapeTimeout, + ); + + it.concurrent( + "should successfully cancel an in-progress crawl job", + async () => { + const startResponse = await crawlStart( + { + url: "https://firecrawl.dev", + limit: 100, + }, + identity, + ); + + expect(startResponse.statusCode).toBe(200); + const crawlId = startResponse.body.id; + + await new Promise(resolve => setTimeout(resolve, 1000)); + + const cancelResponse = await fetch( + `${process.env.TEST_API_URL}/v1/crawl/${crawlId}/cancel`, + { + method: "DELETE", + headers: { + Authorization: `Bearer ${identity.apiKey}`, + "Content-Type": "application/json", + }, + }, + ); + + expect(cancelResponse.status).toBe(200); + const cancelData = await cancelResponse.json(); + expect(cancelData.status).toBe("cancelled"); + }, + scrapeTimeout, + ); + }); }); diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index 78d1cbf52e..4c85add32a 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -2,12 +2,28 @@ import { Request, Response } from "express"; import { authenticateUser } from "../auth"; import { RateLimiterMode } from "../../../src/types"; import { logger } from "../../../src/lib/logger"; -import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis"; +import { + getCrawl, + saveCrawl, + isCrawlKickoffFinished, +} from "../../../src/lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { redisEvictConnection } from "../../../src/services/redis"; configDotenv(); +async function isCrawlFinished(id: string) { + await redisEvictConnection.expire( + "crawl:" + id + ":kickoff:finish", + 24 * 60 * 60, + ); + return ( + (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === + (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && + (await isCrawlKickoffFinished(id)) + ); +} + export async function crawlCancelController(req: Request, res: Response) { try { const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus); @@ -53,6 +69,17 @@ export async function crawlCancelController(req: Request, res: Response) { return res.status(403).json({ error: "Unauthorized" }); } + try { + const isCompleted = await isCrawlFinished(req.params.jobId); + if (isCompleted) { + return res.status(409).json({ + error: "Cannot cancel job that has already completed", + }); + } + } catch (error) { + logger.error("Error checking crawl completion status", error); + } + try { sc.cancelled = true; await saveCrawl(req.params.jobId, sc); diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts index 8ddd9fdba6..f8cb02262b 100644 --- a/apps/api/src/controllers/v1/crawl-cancel.ts +++ b/apps/api/src/controllers/v1/crawl-cancel.ts @@ -1,11 +1,28 @@ import { Response } from "express"; import { logger } from "../../lib/logger"; -import { getCrawl, saveCrawl } from "../../lib/crawl-redis"; +import { + getCrawl, + saveCrawl, + isCrawlKickoffFinished, +} from "../../lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { RequestWithAuth } from "./types"; +import { redisEvictConnection } from "../../services/redis"; configDotenv(); +async function isCrawlFinished(id: string) { + await redisEvictConnection.expire( + "crawl:" + id + ":kickoff:finish", + 24 * 60 * 60, + ); + return ( + (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === + (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && + (await isCrawlKickoffFinished(id)) + ); +} + export async function crawlCancelController( req: RequestWithAuth<{ jobId: string }>, res: Response, @@ -20,6 +37,17 @@ export async function crawlCancelController( return res.status(403).json({ error: "Unauthorized" }); } + try { + const isCompleted = await isCrawlFinished(req.params.jobId); + if (isCompleted) { + return res.status(409).json({ + error: "Cannot cancel job that has already completed", + }); + } + } catch (error) { + logger.error("Error checking crawl completion status", error); + } + try { sc.cancelled = true; await saveCrawl(req.params.jobId, sc); diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts index a74939fb79..93d78c247a 100644 --- a/apps/api/src/controllers/v2/crawl-cancel.ts +++ b/apps/api/src/controllers/v2/crawl-cancel.ts @@ -1,11 +1,28 @@ import { Response } from "express"; import { logger } from "../../lib/logger"; -import { getCrawl, saveCrawl } from "../../lib/crawl-redis"; +import { + getCrawl, + saveCrawl, + isCrawlKickoffFinished, +} from "../../lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { RequestWithAuth } from "./types"; +import { redisEvictConnection } from "../../services/redis"; configDotenv(); +async function isCrawlFinished(id: string) { + await redisEvictConnection.expire( + "crawl:" + id + ":kickoff:finish", + 24 * 60 * 60, + ); + return ( + (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === + (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && + (await isCrawlKickoffFinished(id)) + ); +} + export async function crawlCancelController( req: RequestWithAuth<{ jobId: string }>, res: Response, @@ -21,6 +38,17 @@ export async function crawlCancelController( return res.status(403).json({ error: "Unauthorized" }); } + try { + const isCompleted = await isCrawlFinished(req.params.jobId); + if (isCompleted) { + return res.status(409).json({ + error: "Cannot cancel job that has already completed", + }); + } + } catch (error) { + logger.error("Error checking crawl completion status", error); + } + try { sc.cancelled = true; await saveCrawl(req.params.jobId, sc); diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json index ccab024d58..a70aa27c15 100644 --- a/apps/api/v1-openapi.json +++ b/apps/api/v1-openapi.json @@ -378,6 +378,22 @@ } } }, + "409": { + "description": "Job already completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Cannot cancel job that has already completed" + } + } + } + } + } + }, "500": { "description": "Server error", "content": { @@ -607,6 +623,22 @@ } } }, + "409": { + "description": "Job already completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string", + "example": "Cannot cancel job that has already completed" + } + } + } + } + } + }, "500": { "description": "Server error", "content": { From 974f8038c666e86c7bb507c6459e05ec98bab91f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 07:34:13 +0000 Subject: [PATCH 2/8] Fix code duplication: export isCrawlFinished from crawl-redis and remove duplicated functions - Export isCrawlFinished function from crawl-redis.ts - Remove duplicated isCrawlFinished implementations from v0, v1, v2 controllers - Import the function from crawl-redis.ts instead of duplicating code - Addresses GitHub PR comment from mogery Co-Authored-By: Micah Stairs --- apps/api/src/controllers/v0/crawl-cancel.ts | 14 +------------- apps/api/src/controllers/v1/crawl-cancel.ts | 19 +------------------ apps/api/src/controllers/v2/crawl-cancel.ts | 19 +------------------ apps/api/src/lib/crawl-redis.ts | 2 +- 4 files changed, 4 insertions(+), 50 deletions(-) diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index 4c85add32a..60ecfb4891 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -5,25 +5,13 @@ import { logger } from "../../../src/lib/logger"; import { getCrawl, saveCrawl, - isCrawlKickoffFinished, + isCrawlFinished, } from "../../../src/lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { redisEvictConnection } from "../../../src/services/redis"; configDotenv(); -async function isCrawlFinished(id: string) { - await redisEvictConnection.expire( - "crawl:" + id + ":kickoff:finish", - 24 * 60 * 60, - ); - return ( - (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === - (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && - (await isCrawlKickoffFinished(id)) - ); -} - export async function crawlCancelController(req: Request, res: Response) { try { const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus); diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts index f8cb02262b..0ddb0546e6 100644 --- a/apps/api/src/controllers/v1/crawl-cancel.ts +++ b/apps/api/src/controllers/v1/crawl-cancel.ts @@ -1,28 +1,11 @@ import { Response } from "express"; import { logger } from "../../lib/logger"; -import { - getCrawl, - saveCrawl, - isCrawlKickoffFinished, -} from "../../lib/crawl-redis"; +import { getCrawl, saveCrawl, isCrawlFinished } from "../../lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { RequestWithAuth } from "./types"; -import { redisEvictConnection } from "../../services/redis"; configDotenv(); -async function isCrawlFinished(id: string) { - await redisEvictConnection.expire( - "crawl:" + id + ":kickoff:finish", - 24 * 60 * 60, - ); - return ( - (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === - (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && - (await isCrawlKickoffFinished(id)) - ); -} - export async function crawlCancelController( req: RequestWithAuth<{ jobId: string }>, res: Response, diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts index 93d78c247a..0a56ac9aa0 100644 --- a/apps/api/src/controllers/v2/crawl-cancel.ts +++ b/apps/api/src/controllers/v2/crawl-cancel.ts @@ -1,28 +1,11 @@ import { Response } from "express"; import { logger } from "../../lib/logger"; -import { - getCrawl, - saveCrawl, - isCrawlKickoffFinished, -} from "../../lib/crawl-redis"; +import { getCrawl, saveCrawl, isCrawlFinished } from "../../lib/crawl-redis"; import * as Sentry from "@sentry/node"; import { configDotenv } from "dotenv"; import { RequestWithAuth } from "./types"; -import { redisEvictConnection } from "../../services/redis"; configDotenv(); -async function isCrawlFinished(id: string) { - await redisEvictConnection.expire( - "crawl:" + id + ":kickoff:finish", - 24 * 60 * 60, - ); - return ( - (await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) === - (await redisEvictConnection.scard("crawl:" + id + ":jobs")) && - (await isCrawlKickoffFinished(id)) - ); -} - export async function crawlCancelController( req: RequestWithAuth<{ jobId: string }>, res: Response, diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts index 186835a688..7bcddb7ced 100644 --- a/apps/api/src/lib/crawl-redis.ts +++ b/apps/api/src/lib/crawl-redis.ts @@ -189,7 +189,7 @@ export async function getDoneJobsOrderedUntil( ); } -async function isCrawlFinished(id: string) { +export async function isCrawlFinished(id: string) { await redisEvictConnection.expire( "crawl:" + id + ":kickoff:finish", 24 * 60 * 60, From 008287e32f5765439dfc726df8b0160d976bcedb Mon Sep 17 00:00:00 2001 From: Micah Stairs Date: Fri, 26 Sep 2025 17:14:01 +0300 Subject: [PATCH 3/8] Update apps/api/src/controllers/v0/crawl-cancel.ts Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> --- apps/api/src/controllers/v0/crawl-cancel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index 60ecfb4891..2b7f9ed06e 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -65,7 +65,7 @@ export async function crawlCancelController(req: Request, res: Response) { }); } } catch (error) { - logger.error("Error checking crawl completion status", error); + logger.error("Error checking crawl completion status", { error }); } try { From f590f3f8510f7a9108c8c286b93df0636b4de3e4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:30:57 +0000 Subject: [PATCH 4/8] Address PR feedback: fix v1 cancel URL, improve test reliability, and update logger syntax - Fix v1 cancel URL by removing /cancel suffix (should be DELETE /v1/crawl/{id}) - Replace brittle 1s delays with status checking in tests to avoid flaky outcomes - Update logger.error calls to use object format: { error } instead of passing error directly - Apply fixes across all API versions (v0, v1, v2) for consistency Co-Authored-By: Micah Stairs --- .../__tests__/snips/v1/batch-scrape.test.ts | 17 ++++++++++++++- apps/api/src/__tests__/snips/v1/crawl.test.ts | 21 ++++++++++++++++--- apps/api/src/controllers/v1/crawl-cancel.ts | 2 +- apps/api/src/controllers/v2/crawl-cancel.ts | 2 +- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts index 65e18c3d43..bc54aa253d 100644 --- a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts +++ b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts @@ -155,7 +155,22 @@ describe("Batch scrape tests", () => { const startData = await startResponse.json(); const batchId = startData.id; - await new Promise(resolve => setTimeout(resolve, 1000)); + let attempts = 0; + let statusResponse; + do { + await new Promise(resolve => setTimeout(resolve, 500)); + statusResponse = await fetch( + `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, + { + headers: { + Authorization: `Bearer ${identity.apiKey}`, + }, + }, + ); + const statusData = await statusResponse.json(); + attempts++; + if (statusData.status === "scraping" || attempts >= 10) break; + } while (attempts < 10); const cancelResponse = await fetch( `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts index ede23c27a7..26e8d93bb4 100644 --- a/apps/api/src/__tests__/snips/v1/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts @@ -404,7 +404,7 @@ describe("Crawl tests", () => { const crawlId = res.id; const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/crawl/${crawlId}/cancel`, + `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, { method: "DELETE", headers: { @@ -438,10 +438,25 @@ describe("Crawl tests", () => { expect(startResponse.statusCode).toBe(200); const crawlId = startResponse.body.id; - await new Promise(resolve => setTimeout(resolve, 1000)); + let attempts = 0; + let statusResponse; + do { + await new Promise(resolve => setTimeout(resolve, 500)); + statusResponse = await fetch( + `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, + { + headers: { + Authorization: `Bearer ${identity.apiKey}`, + }, + }, + ); + const statusData = await statusResponse.json(); + attempts++; + if (statusData.status === "scraping" || attempts >= 10) break; + } while (attempts < 10); const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/crawl/${crawlId}/cancel`, + `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, { method: "DELETE", headers: { diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts index 0ddb0546e6..7a8e16c110 100644 --- a/apps/api/src/controllers/v1/crawl-cancel.ts +++ b/apps/api/src/controllers/v1/crawl-cancel.ts @@ -28,7 +28,7 @@ export async function crawlCancelController( }); } } catch (error) { - logger.error("Error checking crawl completion status", error); + logger.error("Error checking crawl completion status", { error }); } try { diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts index 0a56ac9aa0..d68dbb6cda 100644 --- a/apps/api/src/controllers/v2/crawl-cancel.ts +++ b/apps/api/src/controllers/v2/crawl-cancel.ts @@ -29,7 +29,7 @@ export async function crawlCancelController( }); } } catch (error) { - logger.error("Error checking crawl completion status", error); + logger.error("Error checking crawl completion status", { error }); } try { From 8b6a04f60bf49fa1dffa756b4046220cdad0e481 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 20:57:50 +0000 Subject: [PATCH 5/8] Revert test file changes: remove cancel endpoint tests - Remove cancel test suites from crawl.test.ts and batch-scrape.test.ts - Keep controller changes that implement 409 error functionality - Tests were removed per user request while maintaining core feature Co-Authored-By: Micah Stairs --- .../__tests__/snips/v1/batch-scrape.test.ts | 97 ------------------- apps/api/src/__tests__/snips/v1/crawl.test.ts | 87 ----------------- 2 files changed, 184 deletions(-) diff --git a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts index bc54aa253d..23dd503ebe 100644 --- a/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts +++ b/apps/api/src/__tests__/snips/v1/batch-scrape.test.ts @@ -94,101 +94,4 @@ describe("Batch scrape tests", () => { }, scrapeTimeout, ); - - describe("DELETE /v1/batch/scrape/{id}", () => { - it.concurrent( - "should return 409 when trying to cancel a completed batch scrape job", - async () => { - const response = await batchScrape( - { - urls: ["https://example.com"], - }, - identity, - ); - - expect(response.success).toBe(true); - if (response.success) { - const batchId = response.id; - - const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, - { - method: "DELETE", - headers: { - Authorization: `Bearer ${identity.apiKey}`, - "Content-Type": "application/json", - }, - }, - ); - - expect(cancelResponse.status).toBe(409); - const cancelData = await cancelResponse.json(); - expect(cancelData.error).toBe( - "Cannot cancel job that has already completed", - ); - } - }, - scrapeTimeout, - ); - - it.concurrent( - "should successfully cancel an in-progress batch scrape job", - async () => { - const startResponse = await fetch( - `${process.env.TEST_API_URL}/v1/batch/scrape`, - { - method: "POST", - headers: { - Authorization: `Bearer ${identity.apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - urls: Array.from( - { length: 10 }, - (_, i) => `https://firecrawl.dev/page-${i}`, - ), - }), - }, - ); - - expect(startResponse.status).toBe(200); - const startData = await startResponse.json(); - const batchId = startData.id; - - let attempts = 0; - let statusResponse; - do { - await new Promise(resolve => setTimeout(resolve, 500)); - statusResponse = await fetch( - `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, - { - headers: { - Authorization: `Bearer ${identity.apiKey}`, - }, - }, - ); - const statusData = await statusResponse.json(); - attempts++; - if (statusData.status === "scraping" || attempts >= 10) break; - } while (attempts < 10); - - const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/batch/scrape/${batchId}`, - { - method: "DELETE", - headers: { - Authorization: `Bearer ${identity.apiKey}`, - "Content-Type": "application/json", - }, - }, - ); - - expect(cancelResponse.status).toBe(200); - const cancelData = await cancelResponse.json(); - expect(cancelData.success).toBe(true); - expect(cancelData.message).toContain("cancelled"); - }, - scrapeTimeout, - ); - }); }); diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts index 26e8d93bb4..ce0550c66e 100644 --- a/apps/api/src/__tests__/snips/v1/crawl.test.ts +++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts @@ -386,91 +386,4 @@ describe("Crawl tests", () => { expect(typeof response.body.id).toBe("string"); }, ); - - describe("POST /v1/crawl/{id}/cancel", () => { - it.concurrent( - "should return 409 when trying to cancel a completed crawl job", - async () => { - const res = await crawl( - { - url: "https://example.com", - limit: 1, - }, - identity, - ); - - expect(res.success).toBe(true); - if (res.success) { - const crawlId = res.id; - - const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, - { - method: "DELETE", - headers: { - Authorization: `Bearer ${identity.apiKey}`, - "Content-Type": "application/json", - }, - }, - ); - - expect(cancelResponse.status).toBe(409); - const cancelData = await cancelResponse.json(); - expect(cancelData.error).toBe( - "Cannot cancel job that has already completed", - ); - } - }, - scrapeTimeout, - ); - - it.concurrent( - "should successfully cancel an in-progress crawl job", - async () => { - const startResponse = await crawlStart( - { - url: "https://firecrawl.dev", - limit: 100, - }, - identity, - ); - - expect(startResponse.statusCode).toBe(200); - const crawlId = startResponse.body.id; - - let attempts = 0; - let statusResponse; - do { - await new Promise(resolve => setTimeout(resolve, 500)); - statusResponse = await fetch( - `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, - { - headers: { - Authorization: `Bearer ${identity.apiKey}`, - }, - }, - ); - const statusData = await statusResponse.json(); - attempts++; - if (statusData.status === "scraping" || attempts >= 10) break; - } while (attempts < 10); - - const cancelResponse = await fetch( - `${process.env.TEST_API_URL}/v1/crawl/${crawlId}`, - { - method: "DELETE", - headers: { - Authorization: `Bearer ${identity.apiKey}`, - "Content-Type": "application/json", - }, - }, - ); - - expect(cancelResponse.status).toBe(200); - const cancelData = await cancelResponse.json(); - expect(cancelData.status).toBe("cancelled"); - }, - scrapeTimeout, - ); - }); }); From a7aaf0ee5fd53c9058c7751fe6ffdfe46efd1c25 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 21:06:05 +0000 Subject: [PATCH 6/8] Update error message to be more specific - Change error message from 'Cannot cancel job that has already completed' to 'Crawl job already completed' to match OpenAPI error patterns - Updated across all API versions (v0, v1, v2) Co-Authored-By: Micah Stairs --- apps/api/src/controllers/v0/crawl-cancel.ts | 2 +- apps/api/src/controllers/v1/crawl-cancel.ts | 2 +- apps/api/src/controllers/v2/crawl-cancel.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index 2b7f9ed06e..3778f551f8 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -61,7 +61,7 @@ export async function crawlCancelController(req: Request, res: Response) { const isCompleted = await isCrawlFinished(req.params.jobId); if (isCompleted) { return res.status(409).json({ - error: "Cannot cancel job that has already completed", + error: "Crawl job already completed", }); } } catch (error) { diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.ts index 7a8e16c110..9b9ccb5076 100644 --- a/apps/api/src/controllers/v1/crawl-cancel.ts +++ b/apps/api/src/controllers/v1/crawl-cancel.ts @@ -24,7 +24,7 @@ export async function crawlCancelController( const isCompleted = await isCrawlFinished(req.params.jobId); if (isCompleted) { return res.status(409).json({ - error: "Cannot cancel job that has already completed", + error: "Crawl job already completed", }); } } catch (error) { diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts index d68dbb6cda..b537be1306 100644 --- a/apps/api/src/controllers/v2/crawl-cancel.ts +++ b/apps/api/src/controllers/v2/crawl-cancel.ts @@ -25,7 +25,7 @@ export async function crawlCancelController( const isCompleted = await isCrawlFinished(req.params.jobId); if (isCompleted) { return res.status(409).json({ - error: "Cannot cancel job that has already completed", + error: "Crawl job already completed", }); } } catch (error) { From c6f29d8f772e1525fd2652bee4f3eeaee9f67221 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 21:07:18 +0000 Subject: [PATCH 7/8] Update OpenAPI error messages to be more specific - Change error message from 'Cannot cancel job that has already completed' to 'Crawl job already completed' in both openapi.json and v1-openapi.json - Makes error message more specific to match other API error patterns - Updated all 4 occurrences across both OpenAPI specification files Co-Authored-By: Micah Stairs --- apps/api/openapi.json | 4 ++-- apps/api/v1-openapi.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 59aa3bb921..9580cb33b7 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -387,7 +387,7 @@ "properties": { "error": { "type": "string", - "example": "Cannot cancel job that has already completed" + "example": "Crawl job already completed" } } } @@ -632,7 +632,7 @@ "properties": { "error": { "type": "string", - "example": "Cannot cancel job that has already completed" + "example": "Crawl job already completed" } } } diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json index a70aa27c15..f4014e691a 100644 --- a/apps/api/v1-openapi.json +++ b/apps/api/v1-openapi.json @@ -387,7 +387,7 @@ "properties": { "error": { "type": "string", - "example": "Cannot cancel job that has already completed" + "example": "Crawl job already completed" } } } @@ -632,7 +632,7 @@ "properties": { "error": { "type": "string", - "example": "Cannot cancel job that has already completed" + "example": "Crawl job already completed" } } } From 91ee2bfe326181249ab6661c47ad7bbd0bdad970 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 26 Sep 2025 23:31:01 +0000 Subject: [PATCH 8/8] Update OpenAPI 409 response descriptions to be more specific - Changed description from 'Job already completed' to 'Crawl job already completed' - Updated both openapi.json and v1-openapi.json files - Ensures consistency with error message pattern Co-Authored-By: Micah Stairs --- apps/api/openapi.json | 4 ++-- apps/api/v1-openapi.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/openapi.json b/apps/api/openapi.json index 9580cb33b7..a494d47261 100644 --- a/apps/api/openapi.json +++ b/apps/api/openapi.json @@ -379,7 +379,7 @@ } }, "409": { - "description": "Job already completed", + "description": "Crawl job already completed", "content": { "application/json": { "schema": { @@ -624,7 +624,7 @@ } }, "409": { - "description": "Job already completed", + "description": "Crawl job already completed", "content": { "application/json": { "schema": { diff --git a/apps/api/v1-openapi.json b/apps/api/v1-openapi.json index f4014e691a..3001caed96 100644 --- a/apps/api/v1-openapi.json +++ b/apps/api/v1-openapi.json @@ -379,7 +379,7 @@ } }, "409": { - "description": "Job already completed", + "description": "Crawl job already completed", "content": { "application/json": { "schema": { @@ -624,7 +624,7 @@ } }, "409": { - "description": "Job already completed", + "description": "Crawl job already completed", "content": { "application/json": { "schema": {