Information Disclosure

Proof-of-concept to demonstrate potential prior leakage and how the fix mitigates it: Assumptions: - A secret is stored in the repository or environment, e.g. SECRET=supersecret, and the harness loads environment from the parent process by default. - The harness exposes or logs environment variables, enabling potential leakage when the evaluation harness is spawned with inherited env. Pre-fix (demonstrates potential leakage): 1) Create harness.js that prints SECRET from the environment // harness.js console.log('SECRET=' + (process.env.SECRET ?? '(unset)')); 2) From a parent process, run the harness with a secret in its environment: SECRET=supersecret node harness.js 3) Expected outcome (demonstrates leakage): SECRET=supersecret Post-fix (demonstrates mitigation): 1) Create harness.js that loads its own env from a local .env file using dotenv (simulating agent-eval’s new behavior loading env from evals/ directory): // harness.js require('dotenv').config({ path: './env/.env' }) console.log('SECRET=' + (process.env.SECRET ?? '(unset)')); 2) Create env/.env containing the secret for the harness itself (isolated from parent): mkdir -p evals/evals/agent-demo/env printf 'SECRET=localsecret' > evals/evals/agent-demo/env/.env 3) Parent process runs the harness with an externally provided SECRET, but the harness loads its own local env instead of inheriting the parent: SECRET=supersecret node harness.js 4) Expected outcome (no leakage): SECRET=localsecret This PoC demonstrates that, prior to the fix, a child harness could reflect secrets from the parent process (inheriting SECRET). After the fix, the harness loads its own env from its directory, preventing leakage of the parent SECRET.

diff --git a/.gitignore b/.gitignore index 91f4836e0e1560..b1c5c5675d96c8 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ examples/**/out/* # env .env*.local +evals/.env pr-stats.md test-timings.json diff --git a/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts b/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts new file mode 100644 index 00000000000000..d53459ce299a9d --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/EVAL.ts @@ -0,0 +1,61 @@ +/** + * Optimize PPR Shell + * + * Tests whether the agent decomposes a monolithic loading.tsx (which creates + * a single implicit Suspense boundary around the entire page) into granular + * Suspense boundaries — one per dashboard section — so each section can + * stream independently and the PPR shell contains more static content. + * + * Tricky because the starting code uses Next.js's loading.tsx convention, + * which is an implicit Suspense boundary. Agents need to recognize that + * loading.tsx creates an all-or-nothing loading state, and that optimizing + * the PPR shell requires replacing it with per-section Suspense boundaries + * so each section can stream independently. + */ + +import { expect, test } from 'vitest' +import { readFileSync } from 'fs' +import { join } from 'path' + +const appDir = join(process.cwd(), 'app') + +function readFile(name: string): string { + return readFileSync(join(appDir, name), 'utf-8') +} + +test('Page has at least 3 Suspense boundaries', () => { + const page = readFile('page.tsx') + + const suspenseCount = (page.match(/<Suspense[\s>]/g) || []).length + expect(suspenseCount).toBeGreaterThanOrEqual(3) +}) + +test('Each dashboard section has its own Suspense boundary in page.tsx', () => { + const page = readFile('page.tsx') + + // Split page into Suspense blocks: text between each <Suspense and </Suspense> + const suspenseBlocks = page.split(/<Suspense[\s>]/).slice(1) + + const components = ['CardStats', 'RevenueChart', 'LatestInvoices'] + for (const component of components) { + const inOwnBlock = suspenseBlocks.some( + (block) => block.includes(component) && block.includes('</Suspense>') + ) + expect(inOwnBlock, `${component} should be inside its own <Suspense>`).toBe( + true + ) + } +}) + +test('Page does not await all data before rendering', () => { + const page = readFile('page.tsx') + + // The page should not call getDashboardData() or fetch() at the top level. + // A simple check: the page shouldn't contain the original monolithic fetch. + expect(page).not.toMatch(/await\s+getDashboardData\s*\(/) + + // The page component itself should not be async (data fetching moves into children) + // OR if it is async, it should not await a data fetch before returning JSX. + // We check the simpler signal: getDashboardData should not be called in page.tsx at all. + expect(page).not.toMatch(/getDashboardData\s*\(/) +}) diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/CardStats.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/CardStats.tsx new file mode 100644 index 00000000000000..035806a581244c --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/CardStats.tsx @@ -0,0 +1,20 @@ +export function CardStats({ + totalRevenue, + totalInvoices, +}: { + totalRevenue: number + totalInvoices: number +}) { + return ( + <div className="grid grid-cols-2 gap-4"> + <div className="card"> + <h2>Total Revenue</h2> + <p>${totalRevenue.toLocaleString()}</p> + </div> + <div className="card"> + <h2>Total Invoices</h2> + <p>{totalInvoices}</p> + </div> + </div> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/LatestInvoices.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/LatestInvoices.tsx new file mode 100644 index 00000000000000..d8e39c0cb88662 --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/LatestInvoices.tsx @@ -0,0 +1,18 @@ +export function LatestInvoices({ + invoices, +}: { + invoices: { id: string; name: string; amount: number }[] +}) { + return ( + <div className="invoices"> + <h2>Latest Invoices</h2> + <ul> + {invoices.map((invoice) => ( + <li key={invoice.id}> + {invoice.name} - ${invoice.amount.toLocaleString()} + </li> + ))} + </ul> + </div> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/RevenueChart.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/RevenueChart.tsx new file mode 100644 index 00000000000000..8c8402e5a9d53c --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/RevenueChart.tsx @@ -0,0 +1,18 @@ +export function RevenueChart({ + revenue, +}: { + revenue: { month: string; amount: number }[] +}) { + return ( + <div className="chart"> + <h2>Revenue</h2> + <ul> + {revenue.map((item) => ( + <li key={item.month}> + {item.month}: ${item.amount.toLocaleString()} + </li> + ))} + </ul> + </div> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/layout.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/layout.tsx new file mode 100644 index 00000000000000..6b8b4518030f13 --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'Create Next App', + description: 'Generated by create next app', +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + <html lang="en"> + <body>{children}</body> + </html> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/loading.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/loading.tsx new file mode 100644 index 00000000000000..f8b61d8bb77781 --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/loading.tsx @@ -0,0 +1,8 @@ +export default function Loading() { + return ( + <div className="loading"> + <div className="spinner" /> + <p>Loading dashboard...</p> + </div> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/app/page.tsx b/evals/evals/agent-041-optimize-ppr-shell/app/page.tsx new file mode 100644 index 00000000000000..2de5ffb50479f2 --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/app/page.tsx @@ -0,0 +1,24 @@ +import { RevenueChart } from './RevenueChart' +import { LatestInvoices } from './LatestInvoices' +import { CardStats } from './CardStats' + +async function getDashboardData() { + const res = await fetch('https://api.example.com/dashboard') + return res.json() +} + +export default async function Page() { + const data = await getDashboardData() + + return ( + <main> + <h1>Dashboard</h1> + <CardStats + totalRevenue={data.totalRevenue} + totalInvoices={data.totalInvoices} + /> + <RevenueChart revenue={data.revenue} /> + <LatestInvoices invoices={data.invoices} /> + </main> + ) +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/next.config.ts b/evals/evals/agent-041-optimize-ppr-shell/next.config.ts new file mode 100644 index 00000000000000..fa33c7c54f24cc --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/next.config.ts @@ -0,0 +1,7 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = { + cacheComponents: true, +} + +export default nextConfig diff --git a/evals/evals/agent-041-optimize-ppr-shell/package.json b/evals/evals/agent-041-optimize-ppr-shell/package.json new file mode 100644 index 00000000000000..7056dd9ed5f925 --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-041-optimize-ppr-shell/tsconfig.json b/evals/evals/agent-041-optimize-ppr-shell/tsconfig.json new file mode 100644 index 00000000000000..00978ef407fdae --- /dev/null +++ b/evals/evals/agent-041-optimize-ppr-shell/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/evals/evals/agent-042-enable-ppr/EVAL.ts b/evals/evals/agent-042-enable-ppr/EVAL.ts new file mode 100644 index 00000000000000..20606268172d0b --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/EVAL.ts @@ -0,0 +1,31 @@ +/** + * Enable PPR + * + * Tests whether the agent knows that partial pre-rendering is enabled via + * `cacheComponents: true` in next.config.ts, NOT the old + * `experimental: { ppr: true }` flag. + * + * Tricky because most training data and older docs reference the experimental + * flag. The current way to enable PPR in Next.js 16 is `cacheComponents: true`. + */ + +import { expect, test } from 'vitest' +import { readFileSync } from 'fs' +import { join } from 'path' + +test('Enables PPR via cacheComponents in next.config', () => { + const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') + + expect(config).toMatch(/cacheComponents\s*:\s*true/) +}) + +test('Does not use the old experimental.ppr flag', () => { + const config = readFileSync(join(process.cwd(), 'next.config.ts'), 'utf-8') + + // Strip comments to avoid false positives from explanatory comments + const stripped = config + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/\/\/.*$/gm, '') + + expect(stripped).not.toMatch(/ppr\s*:\s*true/) +}) diff --git a/evals/evals/agent-042-enable-ppr/app/ProductList.tsx b/evals/evals/agent-042-enable-ppr/app/ProductList.tsx new file mode 100644 index 00000000000000..748a9909edfe80 --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/app/ProductList.tsx @@ -0,0 +1,18 @@ +async function getProducts() { + const res = await fetch('https://api.example.com/products') + return res.json() +} + +export async function ProductList() { + const products = await getProducts() + + return ( + <ul> + {products.map((p: { id: string; name: string; price: number }) => ( + <li key={p.id}> + {p.name} - ${p.price} + </li> + ))} + </ul> + ) +} diff --git a/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx b/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx new file mode 100644 index 00000000000000..61ee9575d6ad23 --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/app/Recommendations.tsx @@ -0,0 +1,19 @@ +async function getRecommendations() { + const res = await fetch('https://api.example.com/recommendations') + return res.json() +} + +export async function Recommendations() { + const items = await getRecommendations() + + return ( + <div> + <h2>Recommended for you</h2> + <ul> + {items.map((item: { id: string; name: string }) => ( + <li key={item.id}>{item.name}</li> + ))} + </ul> + </div> + ) +} diff --git a/evals/evals/agent-042-enable-ppr/app/layout.tsx b/evals/evals/agent-042-enable-ppr/app/layout.tsx new file mode 100644 index 00000000000000..6b8b4518030f13 --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/app/layout.tsx @@ -0,0 +1,18 @@ +import type { Metadata } from 'next' + +export const metadata: Metadata = { + title: 'Create Next App', + description: 'Generated by create next app', +} + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode +}>) { + return ( + <html lang="en"> + <body>{children}</body> + </html> + ) +} diff --git a/evals/evals/agent-042-enable-ppr/app/page.tsx b/evals/evals/agent-042-enable-ppr/app/page.tsx new file mode 100644 index 00000000000000..9716783285827e --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/app/page.tsx @@ -0,0 +1,17 @@ +import { Suspense } from 'react' +import { ProductList } from './ProductList' +import { Recommendations } from './Recommendations' + +export default function Page() { + return ( + <main> + <h1>Store</h1> + <Suspense fallback={<p>Loading products...</p>}> + <ProductList /> + </Suspense> + <Suspense fallback={<p>Loading recommendations...</p>}> + <Recommendations /> + </Suspense> + </main> + ) +} diff --git a/evals/evals/agent-042-enable-ppr/next.config.ts b/evals/evals/agent-042-enable-ppr/next.config.ts new file mode 100644 index 00000000000000..e4f5738a310b36 --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/next.config.ts @@ -0,0 +1,5 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = {} + +export default nextConfig diff --git a/evals/evals/agent-042-enable-ppr/package.json b/evals/evals/agent-042-enable-ppr/package.json new file mode 100644 index 00000000000000..7056dd9ed5f925 --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/package.json @@ -0,0 +1,23 @@ +{ + "private": true, + "type": "module", + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start" + }, + "dependencies": { + "next": "^16", + "react": "19.1.0", + "react-dom": "19.1.0" + }, + "devDependencies": { + "@types/node": "^20", + "@types/react": "^19", + "@types/react-dom": "^19", + "typescript": "^5", + "vitest": "^3.1.3", + "@vitejs/plugin-react": "^4.4.1", + "vite-tsconfig-paths": "^5.1.4" + } +} diff --git a/evals/evals/agent-042-enable-ppr/tsconfig.json b/evals/evals/agent-042-enable-ppr/tsconfig.json new file mode 100644 index 00000000000000..00978ef407fdae --- /dev/null +++ b/evals/evals/agent-042-enable-ppr/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": ["dom", "dom.iterable", "esnext"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "react-jsx", + "incremental": true, + "plugins": [ + { + "name": "next" + } + ], + "paths": { + "@/*": ["./*"] + } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + " ... [truncated]