Fix/p token (#1305)

* preview-token * Nick: --------- Co-authored-by: rafaelmmiller <[email protected]>
mendableai · Mar 6, 2025 · 60346ec · 60346ec
1 parent 39b1390
commit 60346ec
Show file tree

Hide file tree

Showing 8 changed files with 19 additions and 37 deletions.
diff --git a/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts b/apps/api/src/__tests__/e2e_full_withAuth/index.test.ts
@@ -62,15 +62,6 @@ describe("E2E Tests for API Routes", () => {
       expect(response.body.error).toContain(BLOCKLISTED_URL_MESSAGE);
     });
 
-    // tested on rate limit test
-    // it.concurrent("should return a successful response with a valid preview token", async () => {
-    //   const response = await request(TEST_URL)
-    //     .post("/v0/scrape")
-    //     .set("Authorization", `Bearer this_is_just_a_preview_token`)
-    //     .set("Content-Type", "application/json")
-    //     .send({ url: "https://roastmywebsite.ai" });
-    //   expect(response.statusCode).toBe(200);
-    // }, 30000); // 30 seconds timeout
 
     it.concurrent(
       "should return a successful response with a valid API key",
@@ -1087,19 +1078,6 @@ describe("E2E Tests for API Routes", () => {
       },
       3000,
     );
-
-    // it.concurrent("should return a successful response with a valid API key for crawlWebsitePreview", async () => {
-    //   const response = await request(TEST_URL)
-    //     .post("/v0/crawlWebsitePreview")
-    //     .set("Authorization", `Bearer this_is_just_a_preview_token`)
-    //     .set("Content-Type", "application/json")
-    //     .send({ url: "https://firecrawl.dev" });
-    //   expect(response.statusCode).toBe(200);
-    //   expect(response.body).toHaveProperty("jobId");
-    //   expect(response.body.jobId).toMatch(
-    //     /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$/
-    //   );
-    // });
   });
 
   describe("POST /v0/search", () => {
@@ -1724,15 +1702,15 @@ describe("E2E Tests for API Routes", () => {
         for (let i = 0; i < 5; i++) {
           const response = await request(TEST_URL)
             .post("/v0/scrape")
-            .set("Authorization", `Bearer this_is_just_a_preview_token`)
+            .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
             .set("Content-Type", "application/json")
             .send({ url: "https://www.scrapethissite.com" });
 
           expect(response.statusCode).toBe(200);
         }
         const response = await request(TEST_URL)
           .post("/v0/scrape")
-          .set("Authorization", `Bearer this_is_just_a_preview_token`)
+          .set("Authorization", `Bearer ${process.env.PREVIEW_TOKEN}`)
           .set("Content-Type", "application/json")
           .send({ url: "https://www.scrapethissite.com" });
 

diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
@@ -100,7 +100,8 @@ export async function getACUC(
       ? "auth_credit_usage_chunk_extract"
       : "auth_credit_usage_chunk_test_22_credit_pack_n_extract";
     while (retries < maxRetries) {
-      const client = Math.random() > 0.5 ? supabase_rr_service : supabase_service;
+      const client =
+        Math.random() > 0.5 ? supabase_rr_service : supabase_service;
       ({ data, error } = await client.rpc(
         rpcName,
         { input_key: api_key },
@@ -201,8 +202,11 @@ export async function supaAuthenticateUser(
   let chunk: AuthCreditUsageChunk | null = null;
   let plan: PlanType = "free";
   if (token == "this_is_just_a_preview_token") {
-    throw new Error("Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.");
-
+    throw new Error(
+      "Unauthenticated Playground calls are temporarily disabled due to abuse. Please sign up.",
+    );
+  }
+  if (token == process.env.PREVIEW_TOKEN) {
     if (mode == RateLimiterMode.CrawlStatus) {
       rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);
     } else if (mode == RateLimiterMode.ExtractStatus) {
@@ -297,7 +301,7 @@ export async function supaAuthenticateUser(
   }
 
   const team_endpoint_token =
-    token === "this_is_just_a_preview_token" ? iptoken : teamId;
+    token === process.env.PREVIEW_TOKEN ? iptoken : teamId;
 
   try {
     await rateLimiter.consume(team_endpoint_token);
@@ -327,7 +331,7 @@ export async function supaAuthenticateUser(
   }
 
   if (
-    token === "this_is_just_a_preview_token" &&
+    token === process.env.PREVIEW_TOKEN &&
     (mode === RateLimiterMode.Scrape ||
       mode === RateLimiterMode.Preview ||
       mode === RateLimiterMode.Map ||

diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -24,7 +24,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
 
     const incomingIP = (req.headers["x-forwarded-for"] ||
       req.socket.remoteAddress) as string;
-    const iptoken = incomingIP + "this_is_just_a_preview_token";
+    const iptoken = incomingIP + process.env.PREVIEW_TOKEN;
     const team_id = `preview_${iptoken}`;
 
     if (!auth.success) {

diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e_withAuth/index.test.ts
@@ -55,7 +55,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
     "should return successful response with valid preview token",
     async () => {
       const app = new FirecrawlApp<"v0">({
-        apiKey: "this_is_just_a_preview_token",
+        apiKey: process.env.PREVIEW_TOKEN,
         apiUrl: API_URL,
         version: "v0",
       });

diff --git a/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts b/apps/js-sdk/firecrawl/src/__tests__/v1/e2e_withAuth/index.test.ts
@@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {
   });
 
   test.concurrent('should return successful response with valid preview token', async () => {
-    const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
+    const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://roastmywebsite.ai');
     if (!response.success) {
       throw new Error(response.error);
@@ -365,7 +365,7 @@ describe('FirecrawlApp E2E Tests', () => {
   });
 
   test.concurrent('should return successful response with valid preview token', async () => {
-    const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
+    const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
     const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
     expect(response).not.toBeNull();
     expect(response.links?.length).toBeGreaterThan(0);

diff --git a/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/e2e_withAuth/test.py
@@ -37,7 +37,7 @@ def test_scrape_url_invalid_api_key():
 #     assert "Unexpected error during scrape URL: Status code 403. Firecrawl currently does not support social media scraping due to policy restrictions. We're actively working on building support for it." in str(excinfo.value)
 
 def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token", version='v0')
+    app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'), version='v0')
     response = app.scrape_url('https://roastmywebsite.ai')
     assert response is not None
     assert 'content' in response

diff --git a/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py b/apps/python-sdk/firecrawl/__tests__/v1/e2e_withAuth/test.py
@@ -49,7 +49,7 @@ def test_scrape_url_invalid_api_key():
 #     assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
 
 def test_successful_response_with_valid_preview_token():
-    app = FirecrawlApp(api_url=API_URL, api_key="this_is_just_a_preview_token")
+    app = FirecrawlApp(api_url=API_URL, api_key=os.getenv('PREVIEW_TOKEN'))
     response = app.scrape_url('https://roastmywebsite.ai')
     assert response is not None
     assert "_Roast_" in response['markdown']
@@ -327,7 +327,7 @@ def test_invalid_api_key_on_map():
 #     assert "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions." in str(excinfo.value)
 
 def test_successful_response_with_valid_preview_token_on_map():
-    app = FirecrawlApp(api_key="this_is_just_a_preview_token", api_url=API_URL)
+    app = FirecrawlApp(api_key=os.getenv('PREVIEW_TOKEN'), api_url=API_URL)
     response = app.map_url('https://roastmywebsite.ai')
     assert response is not None
     assert len(response) > 0

diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs
@@ -26,7 +26,7 @@ async fn test_successful_response_with_valid_preview_token() {
     let api_url = env::var("API_URL").unwrap();
     let app = FirecrawlApp::new_selfhosted(
         api_url,
-        Some("this_is_just_a_preview_token"),
+        Some(env::var("PREVIEW_TOKEN").unwrap()),
     )
     .unwrap();
     let result = app