From 35516f5885d9acf02003c0d7506406df1ead8798 Mon Sep 17 00:00:00 2001 From: Dev3 ^~^ Date: Fri, 6 Nov 2026 14:02:07 +0530 Subject: [PATCH 1/3] Use verified Flash-Lite analysis and validate model responses across plans --- .env.local.example | 4 +- components/app/AppShell.tsx | 1 + lib/analysisDeadline.ts | 5 +- lib/server/gemini.ts | 23 ++++++--- lib/server/motionAnalysisSchema.ts | 55 ++++++++++++++++++++++ lib/server/openai.ts | 59 ++---------------------- package.json | 3 +- scripts/verify-analysis-build.mjs | 16 +++++++ tests/e2e/app-performance.spec.ts | 26 ++++++++--- tests/fixtures/motion-contact-sheet.jpg | Bin 0 -> 9705 bytes tests/integration/live-analysis.test.ts | 47 +++++++++++++++++++ tests/unit/analyze-api.test.ts | 38 ++++++++++++++- tests/unit/gemini-retries.test.ts | 11 +++-- 13 files changed, 208 insertions(+), 80 deletions(-) create mode 100644 lib/server/motionAnalysisSchema.ts create mode 100644 scripts/verify-analysis-build.mjs create mode 100644 tests/fixtures/motion-contact-sheet.jpg create mode 100644 tests/integration/live-analysis.test.ts diff --git a/.env.local.example b/.env.local.example index 1fbcac5..e0e0424 100644 --- a/.env.local.example +++ b/.env.local.example @@ -29,8 +29,10 @@ NEXT_PUBLIC_SUPABASE_ANON_KEY= SUPABASE_SERVICE_ROLE_KEY= GEMINI_API_KEY= # Optional upstream Gemini models behind the Flash / Pro plan tiers. Defaults: -# gemini-3.8-flash and gemini-3.1-pro-preview (Pro falls back to Flash on error). +# gemini-2.5-flash-lite and gemini-3.1-pro-preview (Pro falls back to Flash-Lite). GEMINI_FLASH_MODEL= +# Backup lightweight model for transient Flash-Lite failures. +GEMINI_FLASH_FALLBACK_MODEL= GEMINI_PRO_MODEL= OPENAI_API_KEY= RAZORPAY_KEY_ID= diff --git a/components/app/AppShell.tsx b/components/app/AppShell.tsx index 1a3c0e7..4d4678f 100644 --- a/components/app/AppShell.tsx +++ b/components/app/AppShell.tsx @@ -424,6 +424,7 @@ export function AppShell({ const analyzed = await withAnalysisDeadline( (signal) => analyzeViaApi({ frames, planTier: userPlan, signal }), controller.signal, + userPlan === "free" ? 90_000 : 105_000, ); if (operationId !== operationIdRef.current) return; diff --git a/lib/analysisDeadline.ts b/lib/analysisDeadline.ts index 3d05cbf..9fe8298 100644 --- a/lib/analysisDeadline.ts +++ b/lib/analysisDeadline.ts @@ -1,10 +1,11 @@ // The browser deadline includes frame preparation, the API call, and body reads. -// Leave time beyond the server's 50-second model budget for auth and transport. +// Leave time beyond each plan's server model budget for auth and transport. export const ANALYSIS_TIMEOUT_MS = 65_000; export async function withAnalysisDeadline( operation: (signal: AbortSignal) => Promise, signal?: AbortSignal, + timeoutMs = ANALYSIS_TIMEOUT_MS, ): Promise { const controller = new AbortController(); if (signal?.aborted) { @@ -17,7 +18,7 @@ export async function withAnalysisDeadline( timer = setTimeout(() => { reject(new Error("Analysis took too long. Please retry; your frames are still ready.")); controller.abort(); - }, ANALYSIS_TIMEOUT_MS); + }, timeoutMs); onAbort = () => { reject(new DOMException("Analysis cancelled.", "AbortError")); controller.abort(); diff --git a/lib/server/gemini.ts b/lib/server/gemini.ts index 3645aaf..5b0762e 100644 --- a/lib/server/gemini.ts +++ b/lib/server/gemini.ts @@ -11,6 +11,7 @@ import { type OutputFramework, } from "@/lib/contracts/motion"; +import { MOTION_ANALYSIS_SCHEMA } from "./motionAnalysisSchema"; import { ApiError } from "./apiErrors"; import { getServerEnv } from "./env"; import { serverLogger } from "./logger"; @@ -26,7 +27,7 @@ export const GEMINI_MODELS = ["gemini-2.5-flash", "gemini-2.5-pro"] as const; // stay put while GEMINI_FLASH_MODEL / GEMINI_PRO_MODEL can swap the upstream // model without code changes. const DEFAULT_UPSTREAM_MODELS: Record = { - "gemini-2.5-flash": "gemini-3.8-flash", + "gemini-2.5-flash": "gemini-2.5-flash-lite", "gemini-2.5-pro": "gemini-3.1-pro-preview", }; @@ -171,6 +172,7 @@ Use this exact shape: } Code requirements (each snippet must run on its own in a live preview): +- Keep each snippet concise: at most 40 lines, with no external media or long embedded data. - Target the animated element with the class "element" in every snippet. - css: complete rules plus @keyframes; bind the animation (or a transition with its :hover/state rule) to .element. - gsap: ES module that imports gsap from "gsap" and animates ".element" at the top level. @@ -203,16 +205,23 @@ export async function analyzeFramesWithGemini( ): Promise { const apiKey = deps.apiKey ?? getServerEnv().geminiApiKey; const fetcher = deps.fetch ?? fetch; - const deadline = Date.now() + 50_000; + const deadline = Date.now() + (input.model === "gemini-2.5-pro" ? 90_000 : 75_000); const primary = resolveUpstreamGeminiModel(input.model); - const fallback = resolveUpstreamGeminiModel("gemini-2.5-flash"); + const flash = resolveUpstreamGeminiModel("gemini-2.5-flash"); + const fallback = primary === flash + ? process.env.GEMINI_FLASH_FALLBACK_MODEL?.trim() || "gemini-3.1-flash-lite" + : flash; try { - return await requestGeminiWithRetries(input, primary, apiKey, fetcher, deadline); + // Reserve time for a different model rather than spending the entire + // request budget retrying an overloaded primary. + const primaryDeadline = Math.min(deadline, Date.now() + (input.model === "gemini-2.5-pro" ? 60_000 : 45_000)); + return await requestGeminiWithRetries(input, primary, apiKey, fetcher, primaryDeadline); } catch (error) { // Pro models are often unavailable to a key (preview access, free-tier // quota); answer with Flash rather than failing the paid analysis. - if (primary === fallback || !(error instanceof ApiError)) { + if (primary === fallback || !(error instanceof ApiError) || + (input.model === "gemini-2.5-flash" && error.status !== 503)) { throw error; } @@ -245,7 +254,7 @@ async function requestGeminiWithRetries( } const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), Math.min(15_000, remainingMs)); + const timeout = setTimeout(() => controller.abort(), Math.min(30_000, remainingMs)); try { return await requestGeminiAnalysis(input, model, apiKey, fetcher, controller.signal); } catch (error) { @@ -295,6 +304,8 @@ async function requestGeminiAnalysis( ], generationConfig: { maxOutputTokens: 8192, + responseMimeType: "application/json", + responseJsonSchema: MOTION_ANALYSIS_SCHEMA, temperature: 0.1, }, }), diff --git a/lib/server/motionAnalysisSchema.ts b/lib/server/motionAnalysisSchema.ts new file mode 100644 index 0000000..c899674 --- /dev/null +++ b/lib/server/motionAnalysisSchema.ts @@ -0,0 +1,55 @@ +export const MOTION_ANALYSIS_SCHEMA = { + additionalProperties: false, + properties: { + accessibility_note: { type: "string" }, + css: { type: "string" }, + delay_ms: { type: "integer" }, + description: { type: "string" }, + duration_ms: { type: "integer" }, + easing: { type: "string" }, + element: { type: "string" }, + framer_motion: { type: "string" }, + gpu_accelerated: { type: "boolean" }, + gsap: { type: "string" }, + implementation_notes: { + items: { type: "string" }, + type: "array", + }, + intent: { + enum: [ + "entrance", + "exit", + "hover", + "loading", + "loop", + "morph", + "scroll", + "unknown", + ], + type: "string", + }, + keyframes_detected: { type: "integer" }, + loops: { type: "boolean" }, + performance_score: { type: "number" }, + react_spring: { type: "string" }, + }, + required: [ + "intent", + "element", + "duration_ms", + "delay_ms", + "easing", + "loops", + "description", + "keyframes_detected", + "performance_score", + "gpu_accelerated", + "accessibility_note", + "implementation_notes", + "css", + "gsap", + "framer_motion", + "react_spring", + ], + type: "object", +} as const; diff --git a/lib/server/openai.ts b/lib/server/openai.ts index cb33976..b809750 100644 --- a/lib/server/openai.ts +++ b/lib/server/openai.ts @@ -1,3 +1,4 @@ +import { MOTION_ANALYSIS_SCHEMA } from "./motionAnalysisSchema"; import { ApiError } from "./apiErrors"; import { buildGeminiPrompt, @@ -52,7 +53,7 @@ export async function analyzeFramesWithOpenAI( text: { format: { name: "motion_analysis", - schema: OPENAI_MOTION_ANALYSIS_SCHEMA, + schema: MOTION_ANALYSIS_SCHEMA, strict: true, type: "json_schema", }, @@ -75,61 +76,7 @@ export async function analyzeFramesWithOpenAI( return normalizeGeminiGeneratedAnalysis(parsed); } -const OPENAI_MOTION_ANALYSIS_SCHEMA = { - additionalProperties: false, - properties: { - accessibility_note: { type: "string" }, - css: { type: "string" }, - delay_ms: { type: "integer" }, - description: { type: "string" }, - duration_ms: { type: "integer" }, - easing: { type: "string" }, - element: { type: "string" }, - framer_motion: { type: "string" }, - gpu_accelerated: { type: "boolean" }, - gsap: { type: "string" }, - implementation_notes: { - items: { type: "string" }, - type: "array", - }, - intent: { - enum: [ - "entrance", - "exit", - "hover", - "loading", - "loop", - "morph", - "scroll", - "unknown", - ], - type: "string", - }, - keyframes_detected: { type: "integer" }, - loops: { type: "boolean" }, - performance_score: { type: "number" }, - react_spring: { type: "string" }, - }, - required: [ - "intent", - "element", - "duration_ms", - "delay_ms", - "easing", - "loops", - "description", - "keyframes_detected", - "performance_score", - "gpu_accelerated", - "accessibility_note", - "implementation_notes", - "css", - "gsap", - "framer_motion", - "react_spring", - ], - type: "object", -} as const; + function extractOpenAIText(payload: unknown) { const directText = getRecord(payload).output_text; diff --git a/package.json b/package.json index 0aad5ed..e35277a 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "test:watch": "vitest", "test:e2e": "playwright test", "vendor:preview": "node scripts/build-preview-vendor.mjs", - "check": "npm run typecheck && npm run lint && npm test && npm run build" + "check": "npm run typecheck && npm run lint && npm test && npm run build", + "postbuild": "node scripts/verify-analysis-build.mjs" }, "dependencies": { "@base-ui/react": "^1.2.0", diff --git a/scripts/verify-analysis-build.mjs b/scripts/verify-analysis-build.mjs new file mode 100644 index 0000000..7d60911 --- /dev/null +++ b/scripts/verify-analysis-build.mjs @@ -0,0 +1,16 @@ +import { spawnSync } from "node:child_process"; + +// Opt-in release check. Uses the deployment environment's own provider key; +// no credentials are exported and the test adapters do not write user data. +if (process.env.MOTIONCODE_VERIFY_ANALYSIS_ON_BUILD === "true") { + if (!process.env.GEMINI_API_KEY) throw new Error("Missing Gemini key for release verification."); + const result = spawnSync(process.execPath, [ + "node_modules/vitest/vitest.mjs", "run", "tests/integration/live-analysis.test.ts", "--reporter=verbose", + ], { + stdio: "inherit", + env: { ...process.env, MOTIONCODE_LIVE_ANALYSIS: "1" }, + timeout: 360_000, + }); + if (result.error) throw result.error; + if (result.status !== 0) process.exit(result.status ?? 1); +} diff --git a/tests/e2e/app-performance.spec.ts b/tests/e2e/app-performance.spec.ts index 183f082..329f73e 100644 --- a/tests/e2e/app-performance.spec.ts +++ b/tests/e2e/app-performance.spec.ts @@ -163,7 +163,7 @@ test.describe("application performance baseline", () => { ); }); - test("pauses the particle RAF while hidden or outside the viewport", async ({ page }) => { + test("pauses particle rendering while hidden or outside the viewport", async ({ page }) => { await mockSupabaseBrowserRequests(page); await page.addInitScript(() => { const NativeIntersectionObserver = window.IntersectionObserver; @@ -202,12 +202,24 @@ test.describe("application performance baseline", () => { await page.evaluate(() => { let count = 0; let lastAt = performance.now(); - const nativeRequestAnimationFrame = window.requestAnimationFrame.bind(window); - window.requestAnimationFrame = (callback) => { - count += 1; - lastAt = performance.now(); - return nativeRequestAnimationFrame(callback); - }; + // Count this field's actual draw calls. Global requestAnimationFrame also + // includes navigation/scroll animations, which can continue offscreen. + const prototypes = [WebGLRenderingContext.prototype]; + if (typeof WebGL2RenderingContext !== "undefined") { + prototypes.push(WebGL2RenderingContext.prototype); + } + for (const prototype of prototypes) { + for (const method of ["drawArrays", "drawElements"] as const) { + const native = prototype[method]; + Object.defineProperty(prototype, method, { configurable: true, writable: true, value: function (this: WebGLRenderingContext, ...args: number[]) { + if (this.canvas instanceof HTMLCanvasElement && this.canvas.closest('[data-testid="process-field"]')) { + count += 1; + lastAt = performance.now(); + } + return Reflect.apply(native, this, args); + } }); + } + } Object.defineProperty(window, "__motioncodeRafCount", { configurable: true, get: () => count, diff --git a/tests/fixtures/motion-contact-sheet.jpg b/tests/fixtures/motion-contact-sheet.jpg new file mode 100644 index 0000000000000000000000000000000000000000..57cb60578f2d761dd7baa4f0e8d585ec4ff0f46e GIT binary patch literal 9705 zcmeHM2{_bi+keK8-89y$V+~2mp`tKJ$`V3IMIod`*`mVOWjHB95z3M}kyOZ%eXUd0 znxYs>s4O!QgE9a2(K+WmHLmYm-_rZN-}$cou4m?&nfw3u-1q&vx95kvLdamb(GJ5M z0EGeo1^)oj4fFsTnw6Cm%?4lC*w~h_^Kh`k3&YLL$%Do4^J6hstbnkDh=8D&5Ei>a zW`&rfBo2oY5RsLam6Df`!bvSY1O;zpU&hYI!NDgbh!vFjFJDL%z;b}=EJ0|LEMUQ+ z&{z~w2XJtoY^X06`29k$pjqKAIXJnvp+Mnsz=A@fS>XPm0#G^t{tj5NZ2U6ob(aa4 zJFv@o2>uj&=>~_~wt{LQi#Ce9nxkh3Czr6u3Q;kIm5NHrtJF7a+@ztoS#P`kj-3XE zMwWZ5tZnw%?sGcq?BaUF?WmWx&uL%3GiO7?!XqNjM-k)VFDE1>UAdZ^k(rf!Gw0Ur zU+zD6SXfm2sHC)}wywURv8nke3%>rQ7?WODs22-#3oHAgUMLn{xX@TuHktLy_;t5JTw;I!Av^Z&$1d%$(xci zwxsk%mb9x5my|pzh-CQPc0KJp7iXAPwwA+>W$p=g_gZe6%F1 z%e$L5rJ{5*;;<66v?@5^@+60J2u8Y<6## zpbP6EppK-c54?K+9{ajiAw%0#BU}NfLeGlBy_FOrjY{H_ojUrhcVO4^a2yE1XbSb( zXj9b~hLuqW7{&M^Aj3XLvvQfAYt==%dTZecr&@;i&-nW%ZkCjlJqY8uayiM@;Z2&M z_Q$ieHF!+^X#eATl*aJ!JeS>66W%bsIOP$m5OS=?-b7M?TQ95D7XLZbp2F2)?&%7J z*@fkC4@`yC#Ia%z*sM{}hsbMA8GtDq@2F!3UDB1u^Nmrv7D=F4J1a5wld)Q}GS&bs zdsTA!w#1sv7mlC^Om4+}>w{7;C3M;nr>Zl9W8}znzH`hzZDU5qu^SmdniI@u+sO=$ zyAeb%T=_TImxI3^vyHc z6JU_NlA1uk_Q$|W2SXVSIuR&3vbmz5#0adE4=CHgva#&@O}=FMX` z=sKONe@DaE?`XJ?bHkcDM7#d*dx6Fil$06)lbd!u+gGoyHE}yJ9hDnZT=PbLzeI_3 zmxkrfud4yEgJ$UDlSAWc@0u1rKeS22Gv2ez<%*$~N^bjFajC5}J9hDAKOs8-780{3 z{?=joZ%AvoYtIMrQo{3twDi4B^x1K7_O;(VOx!y~OSlxaOOHiFI`p}|?~4}yT?mLf z*-w~9zeT`E(siD-k!kQYQjpdbFE4o8nW6T)tVCON+SSw;BV&FW`S|B1$J`hKv~#!i z2mDf7YcE_DRMt3D?S#HL=y;)oJLZmxvgCQ`NtLz6MWmB~oJTeWO2l=k%uYos(VOpz zNnX0r@$0G&7p$hoxlUi>$7oc?lOs|HnKYrPr(?2`a!oHctUq~7plnqQspCGjWcgUF ze-7@a6YiT1fnx~xQ3n1;<*9}iN$%y?M?=@yZ0r$gy`CnyYRDiv)-1|)#rC>&GM*e- zgEj#NX6jmIS27e4a8J9J5p@>mh$5hQ6Qyw>cnm*fWTSUrrWOwcRxAqSFrth`j(`lJ z(qFD6Imdg$65yfx(qUTSA-t06fLSSfQ9^AIntieJ=q|#@<+~9#y@o4R5}G$r z@e9G@2$(V^!M*-a=7(p#&Ts!WV3tXhYb@aLoifdd z&j#O|c13lP-R*-Y>-@BJH@E$I>P1d7ryU&umMJ2aoG6#y?&};AdikDVk{TdrZkH1$ zXs!>Kmk@J6-;F4lh)SO{p_?J#(7DP5{wcs5&?OTQ1Sqh6QNV10KcTf1r7yS}r7vV- z-Ff_B>c)Vb<|W3*3`D%+@LTa$<;g%(K1egG%V^XiI^LXE=KPCF{kV=+&4Z>QhR7Ki zp6$nHszsBu_-}BqOWybRaP(_qky01(Lm32Uqe~NORy@PeOX*Gw*CKO!nCJeqVkf1z z;jxDMIRzCc z!ITHpfw{imx68`s#Ez$i9j=`|;+Kma2KySyO`&JwOaVI10G&K39vizZcT%-7yvINf zE3Qr1;oQP{Ya6QZ__edi@9LbC9J;I&U@f)7>_jIxQs0+rtx3wDcWGz*1-gd+L>Ek~ zR-X>eLO{juq_-T;C{M5H6`OiyW=m|eYDd8RLn-g}XZj4uRvtpYsxixj@i(q9mp%%S zBI_;c)*N*oPWkyMRYO$CQRYc+Cfzz%U(kAZ`I7mhRE#N;x1uO5aDs`zk7C^50AsNz zz*Lt0H2NRL|MG6RW@SVV=bAMN1`bii&2smw^wIKhet>U^S7ju!&3?-rf1mrsojIYV zt@hJx;>Ue4KJP@?XNr!?^9+k>HvEJlTYVEgXIeH^=eU2t(RgEom!bR%RrZ4ujXs!s5@2$m6vp+;YoAN=t zzZa~bIk|*x%mP+20R62MD*|c52v9KFi-02Boa|<*+Qf_>0-~o}NtA>0F#f9$pmEJa z(EOl$;5DIce+bpMoJ<6C6Be)v9vhgx@FrLGD8qR=&^0}U@W=*+MCHl=eSJZj-t?${ z2E^CC`EJ2slN1P{Hz{jNcx8%T5K=w+^BLcInN5``ypubONqg6JY1XC6NQ$~#kY|C#S8z#=-mV8BfGqLb~PVTci z7Rny|>;P>Y7ocH7szAZd0<{R3<|%^O8y}Qe`kg&Pmu@(R!#4C*Wq&|`oI5nhZM-HI zQ{a8S7V56zKsr6{nG@p`VTAUv=-ZDe;*1GP`r1ito}2|nByC_e^IexxMg>@SsFXnj zbJ1j1cq@+44Q(;2V#74RI)4GC?Qg)uDIBGLJJRG=Rujr#Psv4(t$T2CW)1-ZNkg^Y zW?BCQY74_CtpEYX@l)GdHApl;kEdM{hfzgQy*X9@=RMP@yoOu zJzgt!yl}Gy*K>DLwDF6ZaXLX6M3o6lJw)Vn7UsoNd_T96mK*|Mn5r*H``h$TDh3v< zuky6gjWCN{T{)_VV-OP`?|9T3V=wZU>Kdj~n`00%-s4kpVH0_mmF;QQjJOr#w4ZwxuBTry;>eY(OQi{i51rCwzo*%}>|&$?Cp%Gi{zexXciAIu3O@-hI7c)sAZi zhDH-d=W?p>ld^sUMs;HEoboaE+ed_PulEV3XU|snz8Z5X{I z-tjT~0kJhy0n0neJ6(E*U=w3|{*UsyB=>%oDkNn$Z@idKoDtxl0f&-3g`esJOsLT{ zGZm{Tk*~bpFSqM{UcA16&Plwp=GQ3;Jqdw^t?XdXzs!T@<)`M89s;%MyDSXe8r@%~ zAsyNzJ)M8CMY-8gGr2AEvlJ%JbT7^&y*#D#^Q#LmfLF{_<>&jOVdhhegok~iCt$5i ztgxmO1>dG6*+v_+U;m^SpXgMxJ@ih=XePdiCk{2&yY(L9N$bOfwa-3PG#-XFlEc8>1BGlm8P>X&!i?ne&J*BN!c-24|WCmf6s<*vFZD29WZOe&nhH0 zF5H7*LM5Hpi(l@oTy8=h)7xI0DxKW7r(vAa?#TzTgeOWR=2Oftd$t2HG?lYq3oLqa z57yik+gob2a`bXWZ^n^XVPfGOH&Ur-wodA;ZtcQ1yhR4RxA09N>vScS>q=Pdqc7Y{ zy*lrIIO}r8_!V>geeK>K-|n{vHSNet8yY{|E#cOgJ_tnR(*SFll40>-{y3s%{QaFB zI5d7R5uPwl_ab0t^S;Wj+r0M(z&ungS0j(%w?9f%On$q!X@tY>*U>?q$*rm})N8Kj zY&E7778o2J8qTp}e^#X&m8T)=X#a=K&{EHM`=9SIRdhREuU@8(G4J`hKdRAx^!_iZ zQcLFXe_8SPTCpF>{5To;am@4o<^T>;+al_>jZ`HiSyV!`4risjw{CkXm>g!*RHMLV zpn6evU|BJtPZRQed^(GK-!(kl3V5sAK)#Q&1@e6!kniJx-Fp8mNDU1`zE6V&`MzbR zAm6tK)!KGohOVf?3HiQ8$oHwisz}Ua2&6CaeFKp16NOE3{cd(WeaQD+qbV{Xagc|6 zj$^EfrJ4G}gA=`2$V0k99#UwlB(D(UA-y0EDd7+Qcc=z|_6z=Rv#%xRtxK$6EmDVv zj`&z5C0kSk9ijEf&1QYQ;MuTU&l^^vw(`aq4zL7m3S^qP5BzZ7{|2mnxbH{w{fNGQ NS5p5@(T7J~{{y;g&a(gj literal 0 HcmV?d00001 diff --git a/tests/integration/live-analysis.test.ts b/tests/integration/live-analysis.test.ts new file mode 100644 index 0000000..5ba870b --- /dev/null +++ b/tests/integration/live-analysis.test.ts @@ -0,0 +1,47 @@ +// @vitest-environment node +// Opt in with MOTIONCODE_LIVE_ANALYSIS=1 and a development GEMINI_API_KEY. +// Exercises the real provider and route contract; auth/quota adapters are isolated. +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { expect, test } from "vitest"; +import { handleAnalyzeRequest } from "@/app/api/analyze/handler"; +import { PLAN_ENTITLEMENTS, PLAN_TIERS } from "@/lib/contracts/plans"; +import { analyzeFramesWithGemini } from "@/lib/server/gemini"; +import { createAnalyzeAbuseGuard } from "@/lib/server/abuse"; + +const enabled = process.env.MOTIONCODE_LIVE_ANALYSIS === "1"; +const frame = readFileSync(fileURLToPath(new URL("../fixtures/motion-contact-sheet.jpg", import.meta.url))).toString("base64"); + +for (const planTier of PLAN_TIERS) { + test.skipIf(!enabled)(`live ${planTier} analysis returns usable code`, async () => { + let reservations = 0; + const started = Date.now(); + const model = planTier === "free" ? "gemini-2.5-flash" : "gemini-2.5-pro"; + const response = await handleAnalyzeRequest(new Request("https://motioncode.test/api/analyze", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ frames: [frame], frameGrid: { columns: 4, rows: 1, frameCount: 4 }, model }), + }), { + generateAnalysis: (input) => analyzeFramesWithGemini(input, { apiKey: process.env.GEMINI_API_KEY }), + getCurrentUser: async () => ({ id: `live-test-${planTier}` }), + getEntitlementSummary: async () => ({ planTier, entitlements: PLAN_ENTITLEMENTS[planTier] }), + isOpenAiAnalysisEnabled: () => false, + abuseGuard: createAnalyzeAbuseGuard(), + reserveDailyAnalysisUsage: async () => { reservations += 1; return true; }, + releaseDailyAnalysisUsage: async () => { reservations -= 1; return true; }, + usage: { record: async () => {} }, + audit: { record: async () => {} }, + }); + const json = await response.json(); + console.log(JSON.stringify({ planTier, status: response.status, durationMs: Date.now() - started, code: json.code, outputs: json.data?.outputs?.map((entry: { framework: string }) => entry.framework) })); + expect(response.status, JSON.stringify(json)).toBe(200); + expect(json.ok).toBe(true); + expect(json.data.model).toBe(model); + expect(json.data.frameCount).toBe(4); + expect(reservations).toBe(1); + for (const framework of ["css", "gsap", "framer-motion", "react-spring"]) { + const output = json.data.outputs.find((entry: { framework: string }) => entry.framework === framework); + expect(output?.code.trim().length, framework).toBeGreaterThan(20); + } + }, 120_000); +} diff --git a/tests/unit/analyze-api.test.ts b/tests/unit/analyze-api.test.ts index 03dd202..ee5fd69 100644 --- a/tests/unit/analyze-api.test.ts +++ b/tests/unit/analyze-api.test.ts @@ -707,6 +707,40 @@ describe("POST /api/analyze", () => { expect(deps.generateOpenAiAnalysis).not.toHaveBeenCalled(); }); + it.each(["free", "pro", "studio"] as const)("accepts %s frame limits and rejects excess frames", async (planTier) => { + const { handleAnalyzeRequest } = await import("@/app/api/analyze/handler"); + const deps = createDeps({ planTier }); + const maxFrames = PLAN_ENTITLEMENTS[planTier].maxFramesPerAnalysis; + const model = planTier === "free" ? "gemini-2.5-flash" : "gemini-2.5-pro"; + const body = requestBody({ model, frameGrid: { columns: 4, rows: 4, frameCount: maxFrames } }); + const response = await handleAnalyzeRequest(makeRequest(body), { ...deps, isOpenAiAnalysisEnabled: () => false }); + expect(response.status).toBe(200); + expect((await response.json()).data.frameCount).toBe(maxFrames); + const rejected = await handleAnalyzeRequest(makeRequest({ ...body, frameGrid: { columns: 5, rows: 4, frameCount: maxFrames + 1 } }), deps); + expect(rejected.status).toBe(planTier === "studio" ? 400 : 429); + expect((await rejected.json()).code).toBe(planTier === "studio" ? "INVALID_REQUEST" : "QUOTA_EXCEEDED"); + expect(deps.generateAnalysis).toHaveBeenCalledTimes(1); + }); + + it("lets Free retry after a provider failure and counts only the successful analysis", async () => { + const { handleAnalyzeRequest } = await import("@/app/api/analyze/handler"); + const deps = createDeps({ planTier: "free" }); + let used = 0; + deps.reserveDailyAnalysisUsage.mockImplementation(async () => { + if (used >= 1) return false; + used += 1; + return true; + }); + deps.releaseDailyAnalysisUsage.mockImplementation(async () => { used -= 1; return true; }); + deps.generateAnalysis.mockRejectedValueOnce(new ApiError("MODEL_FAILED", "Busy", 503)); + expect((await handleAnalyzeRequest(makeRequest(requestBody()), deps)).status).toBe(503); + expect(used).toBe(0); + expect((await handleAnalyzeRequest(makeRequest(requestBody()), deps)).status).toBe(200); + expect(used).toBe(1); + expect((await handleAnalyzeRequest(makeRequest(requestBody()), deps)).status).toBe(429); + expect(deps.generateAnalysis).toHaveBeenCalledTimes(2); + }); + it("records usage and audit events with the authorized project workspace when the client omits it", async () => { const { handleAnalyzeRequest } = await import("@/app/api/analyze/handler"); const deps = createDeps(); @@ -1360,7 +1394,7 @@ describe("Gemini upstream models", () => { ); expect(fetcher).toHaveBeenCalledWith( - expect.stringContaining("/gemini-3.8-flash:generateContent"), + expect.stringContaining("/gemini-2.5-flash-lite:generateContent"), expect.anything(), ); }); @@ -1395,7 +1429,7 @@ describe("Gemini upstream models", () => { expect(result.spec.intent).toBe("entrance"); expect(fetcher.mock.calls.map(([url]) => url)).toEqual([ expect.stringContaining("/gemini-3.1-pro-preview:"), - expect.stringContaining("/gemini-3.8-flash:"), + expect.stringContaining("/gemini-2.5-flash-lite:"), ]); }); diff --git a/tests/unit/gemini-retries.test.ts b/tests/unit/gemini-retries.test.ts index 1ffed93..19da5f6 100644 --- a/tests/unit/gemini-retries.test.ts +++ b/tests/unit/gemini-retries.test.ts @@ -14,6 +14,7 @@ beforeEach(() => { vi.useFakeTimers(); vi.stubEnv("GEMINI_FLASH_MODEL", "test-flash"); vi.stubEnv("GEMINI_PRO_MODEL", "test-pro"); + vi.stubEnv("GEMINI_FLASH_FALLBACK_MODEL", "test-lite-backup"); }); afterEach(() => { vi.useRealTimers(); @@ -34,14 +35,14 @@ describe("Gemini transient failures", () => { expect(vi.getTimerCount()).toBe(0); }); - it("stops after three failures and returns a temporary-unavailability error", async () => { + it("stops after bounded failures across both lightweight models and returns a temporary-unavailability error", async () => { const fetcher = vi.fn(async () => busy()); const assertion = expect(run(fetcher)).rejects.toMatchObject({ code: "MODEL_FAILED", status: 503, message: expect.stringContaining("temporarily busy"), }); await vi.runAllTimersAsync(); await assertion; - expect(fetcher).toHaveBeenCalledTimes(3); + expect(fetcher).toHaveBeenCalledTimes(6); }); it.each([400, 401, 403, 404, 429])("does not retry HTTP %s for Flash", async (status) => { @@ -60,7 +61,7 @@ describe("Gemini transient failures", () => { expect(fetcher).toHaveBeenCalledTimes(2); }); - it("aborts stalled requests and bounds primary plus fallback to 50 seconds", async () => { + it("aborts stalled requests and bounds primary plus fallback to 90 seconds", async () => { const started = Date.now(); const fetcher = vi.fn((_url: unknown, options?: RequestInit) => new Promise((_resolve, reject) => { options?.signal?.addEventListener("abort", () => reject(new DOMException("Timed out", "AbortError")), { once: true }); @@ -68,8 +69,8 @@ describe("Gemini transient failures", () => { const assertion = expect(run(fetcher, "gemini-2.5-pro")).rejects.toMatchObject({ code: "MODEL_FAILED", status: 503 }); await vi.runAllTimersAsync(); await assertion; - expect(Date.now() - started).toBeLessThanOrEqual(50_000); - expect(fetcher).toHaveBeenCalledTimes(4); + expect(Date.now() - started).toBeLessThanOrEqual(90_000); + expect(fetcher).toHaveBeenCalledTimes(3); expect(vi.getTimerCount()).toBe(0); }); From d4c680a72e4956cadea8a8397cf8e31deda8f273 Mon Sep 17 00:00:00 2001 From: Dev3 ^~^ Date: Fri, 6 Nov 2026 14:04:17 +0530 Subject: [PATCH 2/3] Run the release analysis checks through the Vercel build lifecycle --- vercel.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vercel.json b/vercel.json index 8e383f7..d1dfa37 100644 --- a/vercel.json +++ b/vercel.json @@ -28,5 +28,6 @@ } ] } - ] + ], + "buildCommand": "npm run build" } From 98b3b8ea94f5bcbc9ccb69d10e9515c460d1a903 Mon Sep 17 00:00:00 2001 From: Dev3 ^~^ Date: Fri, 6 Nov 2026 14:07:04 +0530 Subject: [PATCH 3/3] Use current Flash-Lite for deployment keys and handle retired model fallback --- .env.local.example | 2 +- lib/server/gemini.ts | 6 +++--- tests/unit/analyze-api.test.ts | 4 ++-- tests/unit/gemini-retries.test.ts | 11 ++++++++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.env.local.example b/.env.local.example index e0e0424..e7619f3 100644 --- a/.env.local.example +++ b/.env.local.example @@ -29,7 +29,7 @@ NEXT_PUBLIC_SUPABASE_ANON_KEY= SUPABASE_SERVICE_ROLE_KEY= GEMINI_API_KEY= # Optional upstream Gemini models behind the Flash / Pro plan tiers. Defaults: -# gemini-2.5-flash-lite and gemini-3.1-pro-preview (Pro falls back to Flash-Lite). +# gemini-3.5-flash-lite and gemini-3.1-pro-preview (Pro falls back to Flash-Lite). GEMINI_FLASH_MODEL= # Backup lightweight model for transient Flash-Lite failures. GEMINI_FLASH_FALLBACK_MODEL= diff --git a/lib/server/gemini.ts b/lib/server/gemini.ts index 5b0762e..e3d25f7 100644 --- a/lib/server/gemini.ts +++ b/lib/server/gemini.ts @@ -27,7 +27,7 @@ export const GEMINI_MODELS = ["gemini-2.5-flash", "gemini-2.5-pro"] as const; // stay put while GEMINI_FLASH_MODEL / GEMINI_PRO_MODEL can swap the upstream // model without code changes. const DEFAULT_UPSTREAM_MODELS: Record = { - "gemini-2.5-flash": "gemini-2.5-flash-lite", + "gemini-2.5-flash": "gemini-3.5-flash-lite", "gemini-2.5-pro": "gemini-3.1-pro-preview", }; @@ -221,7 +221,7 @@ export async function analyzeFramesWithGemini( // Pro models are often unavailable to a key (preview access, free-tier // quota); answer with Flash rather than failing the paid analysis. if (primary === fallback || !(error instanceof ApiError) || - (input.model === "gemini-2.5-flash" && error.status !== 503)) { + (input.model === "gemini-2.5-flash" && error.status !== 503 && error.status !== 404)) { throw error; } @@ -337,7 +337,7 @@ async function requestGeminiAnalysis( if (RETRYABLE_STATUSES.has(response.status)) { throw new RetryableGeminiError("MODEL_FAILED", TEMPORARY_FAILURE_MESSAGE, response.status); } - throw new ApiError("MODEL_FAILED", "Gemini analysis failed."); + throw new ApiError("MODEL_FAILED", "Gemini analysis failed.", response.status === 404 ? 404 : 502); } const payload = (await response.json()) as unknown; diff --git a/tests/unit/analyze-api.test.ts b/tests/unit/analyze-api.test.ts index ee5fd69..ed8fef6 100644 --- a/tests/unit/analyze-api.test.ts +++ b/tests/unit/analyze-api.test.ts @@ -1394,7 +1394,7 @@ describe("Gemini upstream models", () => { ); expect(fetcher).toHaveBeenCalledWith( - expect.stringContaining("/gemini-2.5-flash-lite:generateContent"), + expect.stringContaining("/gemini-3.5-flash-lite:generateContent"), expect.anything(), ); }); @@ -1429,7 +1429,7 @@ describe("Gemini upstream models", () => { expect(result.spec.intent).toBe("entrance"); expect(fetcher.mock.calls.map(([url]) => url)).toEqual([ expect.stringContaining("/gemini-3.1-pro-preview:"), - expect.stringContaining("/gemini-2.5-flash-lite:"), + expect.stringContaining("/gemini-3.5-flash-lite:"), ]); }); diff --git a/tests/unit/gemini-retries.test.ts b/tests/unit/gemini-retries.test.ts index 19da5f6..2d4ede1 100644 --- a/tests/unit/gemini-retries.test.ts +++ b/tests/unit/gemini-retries.test.ts @@ -45,13 +45,22 @@ describe("Gemini transient failures", () => { expect(fetcher).toHaveBeenCalledTimes(6); }); - it.each([400, 401, 403, 404, 429])("does not retry HTTP %s for Flash", async (status) => { + it.each([400, 401, 403, 429])("does not retry HTTP %s for Flash", async (status) => { const fetcher = vi.fn(async () => Response.json({ error: { message: "rejected" } }, { status })); await expect(run(fetcher)).rejects.toMatchObject({ code: status === 429 ? "QUOTA_EXCEEDED" : "MODEL_FAILED" }); expect(fetcher).toHaveBeenCalledTimes(1); expect(vi.getTimerCount()).toBe(0); }); + it("uses a compatible lightweight backup when the primary model is retired", async () => { + const fetcher = vi.fn().mockResolvedValueOnce(Response.json({ error: { message: "retired" } }, { status: 404 })) + .mockImplementation(async () => success()); + await expect(run(fetcher)).resolves.toHaveProperty("spec"); + expect(fetcher.mock.calls.map(([url]) => url)).toEqual([ + expect.stringContaining("/test-flash:"), expect.stringContaining("/test-lite-backup:"), + ]); + }); + it("retries a failed network connection", async () => { const fetcher = vi.fn().mockRejectedValueOnce(new TypeError("fetch failed")) .mockImplementation(async () => success());