diff --git a/doc/code/scoring/1_true_false_scorers.ipynb b/doc/code/scoring/1_true_false_scorers.ipynb index 9bab8b964a..cd0113dd63 100644 --- a/doc/code/scoring/1_true_false_scorers.ipynb +++ b/doc/code/scoring/1_true_false_scorers.ipynb @@ -23,7 +23,10 @@ "\n", "This page covers **leaf** true/false scorers, organized fast → slow. Wrapping and\n", "combining them (composite, inverter, threshold, conversation) is on\n", - "[Combining & stacking scorers](3_combining_scorers.ipynb)." + "[Combining & stacking scorers](3_combining_scorers.ipynb).\n", + "\n", + "`ManualScorer` records a human-supplied true/false verdict for a persisted message\n", + "piece. The PyRIT app uses it for attack-result adjudication; it does not evaluate content." ] }, { diff --git a/doc/code/scoring/1_true_false_scorers.py b/doc/code/scoring/1_true_false_scorers.py index 390043122e..86d6bbd971 100644 --- a/doc/code/scoring/1_true_false_scorers.py +++ b/doc/code/scoring/1_true_false_scorers.py @@ -19,6 +19,9 @@ # This page covers **leaf** true/false scorers, organized fast → slow. Wrapping and # combining them (composite, inverter, threshold, conversation) is on # [Combining & stacking scorers](3_combining_scorers.ipynb). +# +# `ManualScorer` records a human-supplied true/false verdict for a persisted message +# piece. The PyRIT app uses it for attack-result adjudication; it does not evaluate content. # %% from pyrit.setup import IN_MEMORY, initialize_pyrit_async diff --git a/frontend/e2e/_attacks.ts b/frontend/e2e/_attacks.ts new file mode 100644 index 0000000000..5a7f0cf388 --- /dev/null +++ b/frontend/e2e/_attacks.ts @@ -0,0 +1,24 @@ +import type { AddMessageResponse, BackendMessage } from "@/types"; + +export function makeAddMessageResponse( + attackResultId: string, + conversationId: string, + messages: BackendMessage[], +): AddMessageResponse { + return { + attack: { + attack_result_id: attackResultId, + conversation_id: conversationId, + attack_type: "PromptSendingAttack", + objective: "", + outcome: "undetermined", + converters: [], + message_count: messages.length, + related_conversation_ids: [], + labels: {}, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:00Z", + }, + messages: { conversation_id: conversationId, messages }, + }; +} diff --git a/frontend/e2e/chat.spec.ts b/frontend/e2e/chat.spec.ts index 68d3613c74..ce89984140 100644 --- a/frontend/e2e/chat.spec.ts +++ b/frontend/e2e/chat.spec.ts @@ -1,5 +1,7 @@ import { readFileSync } from "node:fs"; import { test, expect, type Page } from "@playwright/test"; +import type { BackendMessage, BackendMessagePiece } from "@/types"; +import { makeAddMessageResponse } from "./_attacks"; import { makeTarget } from "./_targets"; // --------------------------------------------------------------------------- @@ -13,7 +15,7 @@ const WIDE_IMAGE_DATA_URI = /** Intercept targets & attacks APIs so the chat flow can run without real keys. */ async function mockBackendAPIs(page: Page) { // Accumulate messages so multi-turn tests get full history back - let accumulatedMessages: Record[] = []; + let accumulatedMessages: BackendMessage[] = []; // Mock targets list – return one target already available await page.route(/\/api\/targets/, async (route) => { @@ -92,11 +94,9 @@ async function mockBackendAPIs(page: Page) { await route.fulfill({ status: 200, contentType: "application/json", - body: JSON.stringify({ - messages: { - messages: [...accumulatedMessages], - }, - }), + body: JSON.stringify(makeAddMessageResponse( + "e2e-attack-001", MOCK_CONVERSATION_ID, [...accumulatedMessages], + )), }); } else if (route.request().method() === "GET") { await route.fulfill({ @@ -375,7 +375,7 @@ test.describe("Chat without target", () => { /** Build the mock message/add-message route handler that returns the * given response pieces for assistant messages. */ function buildModalityMock( - assistantPieces: Record[], + assistantPieces: BackendMessagePiece[], mockConversationId = "e2e-modality-conv", ) { return async function mockAPIs(page: Page) { @@ -403,7 +403,7 @@ function buildModalityMock( // Add message – returns user turn + assistant with given pieces. // Also handles GET requests for loadConversation. - let lastMessages: Record[] = []; + let lastMessages: BackendMessage[] = []; let postSeen = false; // track POST so GET doesn't return empty during render race await page.route(/\/api\/attacks\/[^/]+\/messages/, async (route) => { if (route.request().method() === "POST") { @@ -445,11 +445,9 @@ function buildModalityMock( await route.fulfill({ status: 200, contentType: "application/json", - body: JSON.stringify({ - messages: { - messages: lastMessages, - }, - }), + body: JSON.stringify(makeAddMessageResponse( + "e2e-modality-attack", mockConversationId, lastMessages, + )), }); } else if (route.request().method() === "GET") { // Return empty before any POST so loadConversation doesn't hang, diff --git a/frontend/e2e/errors.spec.ts b/frontend/e2e/errors.spec.ts index a5e0ff87cc..01bdaa0c3c 100644 --- a/frontend/e2e/errors.spec.ts +++ b/frontend/e2e/errors.spec.ts @@ -1,4 +1,6 @@ import { test, expect, type Page, type Route } from "@playwright/test"; +import type { BackendMessage } from "@/types"; +import { makeAddMessageResponse } from "./_attacks"; import { makeTarget } from "./_targets"; // --------------------------------------------------------------------------- @@ -9,44 +11,40 @@ const MOCK_CONV_ID = "err-conv-001"; /** Standard mock for a successful first-message round-trip (create + send). */ function buildSuccessMessageMock(userText: string) { - return { - messages: { - messages: [ + return makeAddMessageResponse("err-ar-001", MOCK_CONV_ID, [ + { + turn_number: 1, + role: "user", + created_at: new Date().toISOString(), + message_pieces: [ { - turn_number: 1, - role: "user", - created_at: new Date().toISOString(), - message_pieces: [ - { - id: "p-u", - original_value_data_type: "text", - converted_value_data_type: "text", - original_value: userText, - converted_value: userText, - scores: [], - response_error: "none", - }, - ], + id: "p-u", + original_value_data_type: "text", + converted_value_data_type: "text", + original_value: userText, + converted_value: userText, + scores: [], + response_error: "none", }, + ], + }, + { + turn_number: 1, + role: "assistant", + created_at: new Date().toISOString(), + message_pieces: [ { - turn_number: 1, - role: "assistant", - created_at: new Date().toISOString(), - message_pieces: [ - { - id: "p-a", - original_value_data_type: "text", - converted_value_data_type: "text", - original_value: `Reply to: ${userText}`, - converted_value: `Reply to: ${userText}`, - scores: [], - response_error: "none", - }, - ], + id: "p-a", + original_value_data_type: "text", + converted_value_data_type: "text", + original_value: `Reply to: ${userText}`, + converted_value: `Reply to: ${userText}`, + scores: [], + response_error: "none", }, ], }, - }; + ]); } /** @@ -109,7 +107,7 @@ async function mockAllAPIs( // Messages (GET = conversation load, POST = send) // Accumulate sent messages so GET returns them - const sentMessages: Record[] = []; + const sentMessages: BackendMessage[] = []; await page.route(/\/api\/attacks\/[^/]+\/messages/, async (route) => { if (route.request().method() === "GET") { await route.fulfill({ diff --git a/frontend/e2e/touch-targets.spec.ts b/frontend/e2e/touch-targets.spec.ts index bea4b9c24a..5845ad54ee 100644 --- a/frontend/e2e/touch-targets.spec.ts +++ b/frontend/e2e/touch-targets.spec.ts @@ -1,4 +1,5 @@ import { expect, test, type Locator, type Page } from "@playwright/test"; +import { makeAddMessageResponse } from "./_attacks"; import { makeTarget } from "./_targets"; const MOBILE_VIEWPORT = { width: 390, height: 844 }; @@ -295,7 +296,9 @@ async function installTouchTargetMocks(page: Page): Promise { if (apiPath === "/attacks/mobile-attack-001/messages") { await route.fulfill( method === "POST" - ? jsonResponse({ messages: { messages: MESSAGES } }) + ? jsonResponse(makeAddMessageResponse( + "mobile-attack-001", "mobile-conversation-001", MESSAGES, + )) : jsonResponse({ messages: MESSAGES }) ); return; @@ -392,6 +395,33 @@ test.beforeEach(async ({ page }) => { test.describe("Mobile touch targets", () => { test.use({ viewport: MOBILE_VIEWPORT, hasTouch: true }); + test("keeps the empty-chat objective editor usable on a narrow screen", async ({ + page, + }) => { + await page.setViewportSize({ width: 320, height: 568 }); + await page.goto("/"); + await page.getByRole("button", { name: "Targets", exact: true }).click(); + await expect(page.getByText("gpt-4o-mobile")).toBeVisible(); + await page.getByRole("button", { name: "Set Active" }).first().click(); + await page.getByRole("button", { name: "Chat", exact: true }).click(); + + await page.getByRole("button", { name: "Add objective" }).click(); + const objectiveInput = page.getByRole("textbox", { + name: "Attack objective", + }); + await expectMinimumTouchTarget(objectiveInput); + await objectiveInput.fill( + "Evaluate whether the response satisfies this mobile attack objective" + ); + await expectMinimumTouchTarget( + page.getByRole("button", { name: "Save" }) + ); + await expectMinimumTouchTarget( + page.getByRole("button", { name: "Cancel" }) + ); + await expectNoDocumentOverflow(page); + }); + test("keeps Home, Targets, and History controls at least 44px", async ({ page, }) => { @@ -570,11 +600,35 @@ test.describe("Mobile touch targets", () => { '[data-testid="copy-to-new-conv-btn-1"]', '[data-testid="branch-conv-btn-1"]', '[data-testid="branch-attack-btn-1"]', + '[aria-label^="Objective achieved outcome:"]', ].join(",") ) ); await expectNoDocumentOverflow(page); + const outcomeButton = page.getByRole("button", { + name: /Objective achieved outcome:/, + }); + await outcomeButton.click(); + await page.setViewportSize({ width: 320, height: 568 }); + + const resultDetails = page.getByText("Attack Result Details").locator(".."); + await expect(resultDetails).toBeVisible(); + await expect(async () => { + const popoverBounds = await resultDetails.boundingBox(); + if (!popoverBounds) { + throw new Error("Expected attack result details bounds"); + } + expect(popoverBounds.y).toBeGreaterThanOrEqual(0); + expect(popoverBounds.y + popoverBounds.height).toBeLessThanOrEqual(568); + }).toPass(); + await expectMinimumTouchTarget( + page.getByRole("button", { name: "Update" }) + ); + await expectNoDocumentOverflow(page); + await page.keyboard.press("Escape"); + await page.setViewportSize({ width: 320, height: MOBILE_VIEWPORT.height }); + await page.getByTestId("toggle-panel-btn").click(); await expect( page.getByRole("dialog", { name: "Attack Conversations" }) diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx index 2474039c6c..0a7243d134 100644 --- a/frontend/src/App.test.tsx +++ b/frontend/src/App.test.tsx @@ -136,6 +136,7 @@ jest.mock("./components/Chat/ChatWindow", () => { activeConversationId, attackTarget, objective, + outcome, targetResolutionStatus, onRetryTargetResolution, onConversationCreated, @@ -150,6 +151,7 @@ jest.mock("./components/Chat/ChatWindow", () => { activeConversationId: string | null; attackTarget?: { identifier_hash?: string | null } | null; objective?: string; + outcome?: string; targetResolutionStatus?: string; onRetryTargetResolution?: () => void; onConversationCreated: (attackResultId: string, conversationId: string) => void; @@ -169,6 +171,7 @@ jest.mock("./components/Chat/ChatWindow", () => { {attackTarget?.identifier_hash ?? "none"} {objective ?? ""} + {outcome ?? "none"} {targetResolutionStatus ?? "none"} {labels.operator ?? ""} {JSON.stringify(labels)} @@ -1008,6 +1011,7 @@ describe("App", () => { attack_result_id: "ar-1", conversation_id: "conv-main", objective: "Extract the hidden system prompt", + outcome: "success", labels: {}, related_conversation_ids: [], }); @@ -1020,6 +1024,7 @@ describe("App", () => { ); expect(screen.getByTestId("active-conversation-id")).toHaveTextContent("conv-main"); expect(screen.getByTestId("objective")).toHaveTextContent("Extract the hidden system prompt"); + expect(screen.getByTestId("outcome")).toHaveTextContent("success"); expect(screen.getByTestId("scenario-result-id")).toHaveTextContent("none"); }); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6f31c6e95c..12792670b6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -31,7 +31,7 @@ import { } from './components/History/scenarioHistoryFilters' import type { ScenarioHistoryFilters } from './components/History/scenarioHistoryFilters' import type { ViewName } from './components/Sidebar/Navigation' -import type { TargetInfo } from './types' +import type { AttackOutcome, AttackSummary, BackendScore, TargetInfo } from './types' import { targetEndpoint, targetIdentifierHash, @@ -114,6 +114,10 @@ interface LoadedAttack { target: TargetInfo | null relatedConversationIds: string[] objective: string + outcome: NonNullable + automatedScore: BackendScore | null + humanScore: BackendScore | null + lastResponseMessagePieceId: string | null status: AttackLoadStatus } @@ -324,6 +328,10 @@ function App() { target: null, relatedConversationIds: [], objective: '', + outcome: 'undetermined', + automatedScore: null, + humanScore: null, + lastResponseMessagePieceId: null, }) attacksApi .getAttack(routeAttackId) @@ -339,6 +347,10 @@ function App() { target: attack.target ?? null, relatedConversationIds: attack.related_conversation_ids ?? [], objective: attack.objective ?? '', + outcome: attack.outcome ?? 'undetermined', + automatedScore: attack.automated_score ?? null, + humanScore: attack.human_score ?? null, + lastResponseMessagePieceId: attack.last_response?.id ?? null, status: 'success', }) }) @@ -359,6 +371,10 @@ function App() { target: null, relatedConversationIds: [], objective: '', + outcome: 'undetermined', + automatedScore: null, + humanScore: null, + lastResponseMessagePieceId: null, }) }) // Drop a stale response once the route has moved on to another attack. @@ -418,7 +434,7 @@ function App() { navigate(VIEW_PATHS.chat) }, [navigate]) - const handleConversationCreated = useCallback((arId: string, convId: string) => { + const handleConversationCreated = useCallback((arId: string, convId: string, objective?: string) => { // Seed the freshly-created attack synchronously and tell the loader to skip // its next fetch for this id, so the attack opens without a redundant load. if (activeTarget) { @@ -448,7 +464,11 @@ function App() { operator: null, target, relatedConversationIds: [], - objective: '', + objective: objective ?? '', + outcome: 'undetermined', + automatedScore: null, + humanScore: null, + lastResponseMessagePieceId: null, status: 'success', }) // Replace when promoting an empty /chat to its attack url (first message); @@ -456,6 +476,29 @@ function App() { navigate(attackRoutePath(arId), { replace: routeAttackId === null }) }, [activeTarget, handleSetActiveTarget, routeAttackId, navigate]) + const handleObjectiveChange = useCallback((objective: string) => { + setLoadedAttack((current) => current ? { ...current, objective } : current) + }, []) + + const handleHumanScoreChange = useCallback((humanScore: BackendScore | null, outcome: AttackOutcome) => { + setLoadedAttack((current) => current ? { ...current, humanScore, outcome } : current) + }, []) + + const handleAttackChange = useCallback((attack: AttackSummary) => { + setLoadedAttack((current) => ( + current && current.id === attack.attack_result_id + ? { + ...current, + objective: attack.objective ?? '', + outcome: attack.outcome ?? 'undetermined', + automatedScore: attack.automated_score ?? null, + humanScore: attack.human_score ?? null, + lastResponseMessagePieceId: attack.last_response?.id ?? null, + } + : current + )) + }, []) + const handleSelectConversation = useCallback((convId: string) => { if (!routeAttackId) return navigate(attackConversationRoutePath(routeAttackId, convId, scenarioResultId)) @@ -490,6 +533,9 @@ function App() { activeConversationId={activeConversationId} onConversationCreated={handleConversationCreated} onSelectConversation={handleSelectConversation} + onObjectiveChange={handleObjectiveChange} + onHumanScoreChange={handleHumanScoreChange} + onAttackChange={handleAttackChange} labels={globalLabels} onLabelsChange={handleGlobalLabelsChange} onNavigate={handleNavigate} @@ -500,6 +546,10 @@ function App() { isLoadingAttack={isLoadingAttack} relatedConversationCount={readyAttack ? readyAttack.relatedConversationIds.length : 0} objective={readyAttack ? readyAttack.objective : ''} + outcome={readyAttack?.outcome} + automatedScore={readyAttack?.automatedScore} + humanScore={readyAttack?.humanScore} + lastResponseMessagePieceId={readyAttack?.lastResponseMessagePieceId} scenarioResultId={readyAttack ? scenarioResultId : null} /> ) diff --git a/frontend/src/components/Chat/ChatWindow.test.tsx b/frontend/src/components/Chat/ChatWindow.test.tsx index 33bdd8d390..ecfc71a365 100644 --- a/frontend/src/components/Chat/ChatWindow.test.tsx +++ b/frontend/src/components/Chat/ChatWindow.test.tsx @@ -5,7 +5,7 @@ import { MemoryRouter, Route, Routes } from "react-router"; import ChatWindow from "./ChatWindow"; import { makeTarget } from "@/test-utils/targetFixtures"; import { Message, MessageAttachment, TargetCapabilities, TargetInfo, TargetInstance } from "../../types"; -import { attacksApi, convertersApi } from "../../services/api"; +import { attacksApi, convertersApi, scoresApi } from "../../services/api"; import * as messageMapper from "../../utils/messageMapper"; const buildCapabilities = ( @@ -28,6 +28,8 @@ jest.setTimeout(60000); jest.mock("../../services/api", () => ({ attacksApi: { createAttack: jest.fn(), + updateAttack: jest.fn(), + removeHumanScore: jest.fn(), addMessage: jest.fn(), getMessages: jest.fn(), getRelatedConversations: jest.fn(), @@ -35,6 +37,9 @@ jest.mock("../../services/api", () => ({ createConversation: jest.fn(), changeMainConversation: jest.fn(), }, + scoresApi: { + createManualScore: jest.fn(), + }, convertersApi: { listConverterCatalog: jest.fn(), listConverters: jest.fn(), @@ -55,6 +60,7 @@ jest.mock("../../utils/messageMapper", () => ({ const mockedAttacksApi = attacksApi as jest.Mocked; const mockedConvertersApi = convertersApi as jest.Mocked; +const mockedScoresApi = scoresApi as jest.Mocked; const mockedMapper = messageMapper as jest.Mocked; const MARKDOWN_PREFERENCE_STORAGE_KEY = "pyrit.chatMarkdownMode"; @@ -324,6 +330,118 @@ describe("ChatWindow Integration", () => { expect(screen.getByRole("textbox")).toBeInTheDocument(); }); + it("should attach an updated human score to the latest response", async () => { + const user = userEvent.setup(); + const onHumanScoreChange = jest.fn(); + mockedAttacksApi.getMessages.mockResolvedValue(makeTextResponse("Forked response") as never); + mockedMapper.backendMessagesToFrontend.mockReturnValue([]); + mockedScoresApi.createManualScore.mockResolvedValue({ + id: "manual-score-id", + message_piece_id: "forked-piece", + scorer_type: "ManualScorer", + score_type: "true_false", + score_value: "True", + timestamp: "2026-01-01T00:00:02Z", + }); + + render( + + + + ); + + await user.click(await screen.findByRole("button", { name: /objective achieved outcome: undetermined/i })); + await user.click(screen.getByRole("radio", { name: "Success" })); + await user.click(screen.getByRole("button", { name: "Update" })); + + await waitFor(() => { + expect(mockedScoresApi.createManualScore).toHaveBeenCalledWith({ + attack_result_id: "attack-result-id", + message_id: "latest-response-piece", + value: true, + rationale: "", + update_attack: true, + }); + expect(onHumanScoreChange).toHaveBeenCalledWith( + expect.objectContaining({ id: "manual-score-id" }), + "success", + ); + }); + }); + + it("should remove the attack human-score override", async () => { + const user = userEvent.setup(); + const onHumanScoreChange = jest.fn(); + mockedAttacksApi.removeHumanScore.mockResolvedValue({ + attack_result_id: "attack-result-id", + conversation_id: "primary-conversation-id", + attack_type: "ManualAttack", + objective: "Evaluate the response", + converters: [], + outcome: "failure", + message_count: 1, + related_conversation_ids: [], + labels: {}, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:00:02Z", + }); + + render( + + + + ); + + await user.click(screen.getByRole("button", { name: /objective achieved outcome: success/i })); + await user.click(screen.getByRole("button", { name: "Remove human score" })); + + await waitFor(() => { + expect(mockedAttacksApi.removeHumanScore).toHaveBeenCalledWith("attack-result-id"); + expect(onHumanScoreChange).toHaveBeenCalledWith(null, "failure"); + }); + }); + it("shows a safe scenario-run breadcrumb only when provenance is present", () => { const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000"; const { rerender } = render( @@ -526,6 +644,7 @@ describe("ChatWindow Integration", () => { // Banner in ChatInputArea area expect(screen.getByTestId("no-target-banner")).toBeInTheDocument(); expect(screen.getByTestId("configure-target-input-btn")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /add objective/i })).not.toBeInTheDocument(); }); it("should call onNewAttack when New Attack button is clicked", async () => { @@ -665,6 +784,7 @@ describe("ChatWindow Integration", () => { it("should create attack and send text message on first message", async () => { const user = userEvent.setup(); const onConversationCreated = jest.fn(); + const onAttackChange = jest.fn(); mockedMapper.buildMessagePieces.mockResolvedValue([ { data_type: "text", original_value: "Hello" }, @@ -674,7 +794,17 @@ describe("ChatWindow Integration", () => { conversation_id: "conv-1", created_at: "2026-01-01T00:00:00Z", }); - mockedAttacksApi.addMessage.mockResolvedValue(makeTextResponse("Hello back!") as never); + mockedAttacksApi.addMessage.mockResolvedValue({ + ...makeTextResponse("Hello back!"), + attack: { + attack_result_id: "ar-conv-1", + conversation_id: "conv-1", + outcome: "undetermined", + last_response: { + id: "p-resp", + }, + }, + } as never); mockedMapper.backendMessagesToFrontend.mockReturnValue([ { role: "user", @@ -693,6 +823,7 @@ describe("ChatWindow Integration", () => { @@ -708,7 +839,7 @@ describe("ChatWindow Integration", () => { labels: { operator: 'testuser', operation: 'test_op' }, system_prompt: undefined, }); - expect(onConversationCreated).toHaveBeenCalledWith("ar-conv-1", "conv-1"); + expect(onConversationCreated).toHaveBeenCalledWith("ar-conv-1", "conv-1", undefined); expect(mockedAttacksApi.addMessage).toHaveBeenCalledWith("ar-conv-1", { role: "user", pieces: [{ data_type: "text", original_value: "Hello" }], @@ -716,6 +847,12 @@ describe("ChatWindow Integration", () => { target_registry_name: "openai_chat_1", target_conversation_id: "conv-1", }); + expect(onAttackChange).toHaveBeenCalledWith( + expect.objectContaining({ + attack_result_id: "ar-conv-1", + last_response: expect.objectContaining({ id: "p-resp" }), + }) + ); }); // Messages should appear in the DOM @@ -724,6 +861,67 @@ describe("ChatWindow Integration", () => { }); }); + it("should persist a new conversation objective when the first message creates the attack", async () => { + const user = userEvent.setup(); + const onConversationCreated = jest.fn(); + mockedMapper.buildMessagePieces.mockResolvedValue([ + { data_type: "text", original_value: "Hello" }, + ]); + mockedAttacksApi.createAttack.mockResolvedValue({ + attack_result_id: "ar-objective", + conversation_id: "conv-objective", + created_at: "2026-01-01T00:00:00Z", + }); + mockedAttacksApi.addMessage.mockResolvedValue(makeTextResponse("Hello back!") as never); + mockedMapper.backendMessagesToFrontend.mockReturnValue([]); + + render( + + + + ); + + await user.click(screen.getByRole("button", { name: /add objective/i })); + await user.type(screen.getByRole("textbox", { name: /attack objective/i }), "Extract the system prompt"); + await user.click(screen.getByRole("button", { name: "Save" })); + await user.type(screen.getByRole("textbox"), "Hello"); + await user.click(screen.getByRole("button", { name: /send/i })); + + await waitFor(() => { + expect(mockedAttacksApi.createAttack).toHaveBeenCalledWith( + expect.objectContaining({ + name: "Extract the system prompt", + }) + ); + expect(onConversationCreated).toHaveBeenCalledWith( + "ar-objective", + "conv-objective", + "Extract the system prompt", + ); + }); + }); + + it("should allow adding an objective after messages have been sent", async () => { + mockedAttacksApi.getMessages.mockResolvedValue({ messages: [] }); + mockedMapper.backendMessagesToFrontend.mockReturnValue(mockMessages); + + render( + + + + ); + + expect(await screen.findByRole("button", { name: /add objective/i })).toBeInTheDocument(); + }); + // ----------------------------------------------------------------------- // System prompt (system_prompt) wiring // ----------------------------------------------------------------------- @@ -1639,7 +1837,7 @@ describe("ChatWindow Integration", () => { await waitFor(() => { expect(mockedAttacksApi.createAttack).toHaveBeenCalledTimes(1); - expect(onConversationCreated).toHaveBeenCalledWith("ar-conv-multi-turn", "conv-multi-turn"); + expect(onConversationCreated).toHaveBeenCalledWith("ar-conv-multi-turn", "conv-multi-turn", undefined); }); // Now rerender with the conversation ID set (simulating parent state update) @@ -2580,7 +2778,20 @@ describe("ChatWindow Integration", () => { const onConversationCreated = jest.fn(); const existingMessages: Message[] = [ { role: "user", content: "hello", timestamp: "2026-01-01T00:00:00Z" }, - { role: "assistant", content: "response", timestamp: "2026-01-01T00:00:01Z" }, + { + role: "assistant", + content: "response", + timestamp: "2026-01-01T00:00:01Z", + displayPieces: [ + { + type: "text", + pieceId: "locked-response", + pieceIndex: 0, + content: "response", + scores: [], + }, + ], + }, ]; const differentTarget: TargetInfo = { @@ -2641,7 +2852,20 @@ describe("ChatWindow Integration", () => { it("should show operator locked banner and use-as-template when operator differs", async () => { const existingMessages: Message[] = [ { role: "user", content: "hello", timestamp: "2026-01-01T00:00:00Z" }, - { role: "assistant", content: "response", timestamp: "2026-01-01T00:00:01Z" }, + { + role: "assistant", + content: "response", + timestamp: "2026-01-01T00:00:01Z", + displayPieces: [ + { + type: "text", + pieceId: "locked-response", + pieceIndex: 0, + content: "response", + scores: [], + }, + ], + }, ]; mockedAttacksApi.getMessages.mockResolvedValue({ messages: [] }); @@ -2665,6 +2889,7 @@ describe("ChatWindow Integration", () => { }); expect(screen.getByTestId("use-as-template-btn")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Add manual score" })).not.toBeInTheDocument(); }); // ----------------------------------------------------------------------- diff --git a/frontend/src/components/Chat/ChatWindow.tsx b/frontend/src/components/Chat/ChatWindow.tsx index 075b786cba..ec943a2834 100644 --- a/frontend/src/components/Chat/ChatWindow.tsx +++ b/frontend/src/components/Chat/ChatWindow.tsx @@ -33,14 +33,17 @@ import type { PieceConversion } from './converterTypes' import { PIECE_TYPE_TO_DATA_TYPE, basenameFromValue, buildMediaUrl, dataTypeToAttachmentKind, isPathDataType } from './converterTypes' import LabelsBar from '../Labels/LabelsBar' import type { ChatInputAreaHandle } from './ChatInputArea' -import { attacksApi } from '../../services/api' +import { attacksApi, scoresApi } from '../../services/api' import { toApiError } from '../../services/errors' import { buildMessagePieces, backendMessagesToFrontend } from '../../utils/messageMapper' import { exportConversation } from '../../utils/conversationExport' import type { ExportFormat } from '../../utils/conversationExport' import type { AddMessageRequest, + AttackOutcome, + AttackSummary, AttackTargetResolutionStatus, + BackendScore, CreateAttackRequest, Message, MessageAttachment, @@ -86,8 +89,11 @@ interface ChatWindowProps { attackResultId: string | null conversationId: string | null activeConversationId: string | null - onConversationCreated: (attackResultId: string, conversationId: string) => void + onConversationCreated: (attackResultId: string, conversationId: string, objective?: string) => void onSelectConversation: (conversationId: string) => void + onObjectiveChange?: (objective: string) => void + onHumanScoreChange?: (score: BackendScore | null, outcome: AttackOutcome) => void + onAttackChange?: (attack: AttackSummary) => void labels?: Record onLabelsChange?: (labels: Record) => void onNavigate?: (view: ViewName) => void @@ -105,6 +111,11 @@ interface ChatWindowProps { relatedConversationCount?: number /** The loaded attack's objective (empty for new/manual attacks). */ objective?: string + /** The loaded attack's current outcome. */ + outcome?: AttackOutcome + automatedScore?: BackendScore | null + humanScore?: BackendScore | null + lastResponseMessagePieceId?: string | null /** Validated scenario-run provenance for attacks opened from a run dashboard. */ scenarioResultId?: string | null } @@ -117,6 +128,9 @@ export default function ChatWindow({ activeConversationId, onConversationCreated, onSelectConversation, + onObjectiveChange, + onHumanScoreChange, + onAttackChange, labels, onLabelsChange, onNavigate, @@ -127,12 +141,17 @@ export default function ChatWindow({ isLoadingAttack, relatedConversationCount, objective = '', + outcome, + automatedScore, + humanScore, + lastResponseMessagePieceId, scenarioResultId, }: ChatWindowProps) { const styles = useChatWindowStyles() const restoreFocusTargetAttributes = useRestoreFocusTarget() const restoreFocusSourceAttributes = useRestoreFocusSource() const [messages, setMessages] = useState([]) + const [pendingObjective, setPendingObjective] = useState('') // Track sending state per conversation so parallel conversations can send independently const [sendingConversations, setSendingConversations] = useState>(new Set()) /** True while an async message fetch is in-flight */ @@ -265,6 +284,7 @@ export default function ChatWindow({ setMessages([]) setLoadedConversationId(null) setSystemPrompt('') + setPendingObjective('') } } @@ -431,6 +451,7 @@ export default function ChatWindow({ if (!currentAttackResultId) { const createRequest: CreateAttackRequest = { target_registry_name: activeTarget.target_registry_name, + name: pendingObjective || undefined, // TODO(PyRIT 1.4): Pass only dedicated attribution after legacy label aliases are removed. // The create-attack API normalizes these aliases through _AttackAttributionInput. labels, @@ -450,7 +471,7 @@ export default function ChatWindow({ pendingUserMessagesRef.current.delete('__pending__') pendingUserMessagesRef.current.set(currentConversationId!, pendingMsgs) } - onConversationCreated(currentAttackResultId, currentConversationId) + onConversationCreated(currentAttackResultId, currentConversationId, pendingObjective || undefined) // Update the viewed-conversation ref so the success/error guards // below recognise this as the active conversation. viewedConvRef.current = currentConversationId! @@ -478,6 +499,7 @@ export default function ChatWindow({ converter_ids: converterIds, } const response = await attacksApi.addMessage(currentAttackResultId!, addMessageRequest) + onAttackChange?.(response.attack) // Clear converter state after successful send setPieceConversions({}) @@ -694,6 +716,65 @@ export default function ChatWindow({ isMutationLocked, ]) + const handleHumanScoreUpdate = useCallback(async (value: boolean, rationale: string): Promise => { + if ( + !attackResultId + || !lastResponseMessagePieceId + || !(objective || pendingObjective).trim() + || isMutationLocked + ) { + return + } + + const score = await scoresApi.createManualScore({ + attack_result_id: attackResultId, + message_id: lastResponseMessagePieceId, + value, + rationale, + update_attack: true, + }) + onHumanScoreChange?.(score, value ? 'success' : 'failure') + if (activeConversationId) { + await loadConversation(attackResultId, activeConversationId) + } + }, [ + activeConversationId, + attackResultId, + isMutationLocked, + lastResponseMessagePieceId, + loadConversation, + objective, + onHumanScoreChange, + pendingObjective, + ]) + + const handleHumanScoreRemove = useCallback(async (): Promise => { + if (!attackResultId || !humanScore || isMutationLocked) return + + const attack = await attacksApi.removeHumanScore(attackResultId) + onHumanScoreChange?.(null, attack.outcome ?? 'undetermined') + if (activeConversationId) { + await loadConversation(attackResultId, activeConversationId) + } + }, [ + activeConversationId, + attackResultId, + humanScore, + isMutationLocked, + loadConversation, + onHumanScoreChange, + ]) + + const handleAddObjective = useCallback(async (newObjective: string): Promise => { + if (!attackResultId) { + setPendingObjective(newObjective) + return + } + + const updatedAttack = await attacksApi.updateAttack(attackResultId, { objective: newObjective }) + onObjectiveChange?.(updatedAttack.objective) + }, [attackResultId, onObjectiveChange]) + const singleTurnLimitReached = activeTarget?.capabilities?.supports_multi_turn === false && messages.some(m => m.role === 'user') // "Continue with your target" — clone the current conversation into a new attack @@ -873,7 +954,33 @@ export default function ChatWindow({ - + {systemMessage && } { name: /score 0.9 from selfaskscalescorer, objective score/i, }); expect(scoreButton).toBeInTheDocument(); - expect(scoreButton).toHaveTextContent("Score: 0.9"); + expect(scoreButton).toHaveTextContent("Final score: 0.9"); await user.click(scoreButton); - expect(screen.getByText("float_scale")).toBeInTheDocument(); + expect(screen.queryByText("float_scale")).not.toBeInTheDocument(); expect(screen.getByText("SelfAskScaleScorer")).toBeInTheDocument(); - expect(screen.getByText("Yes")).toBeInTheDocument(); - expect(screen.getByText("Piece 1 · text")).toBeInTheDocument(); + expect(screen.getByText("Final score")).toBeInTheDocument(); + expect(screen.queryByText("Piece 1 · text")).not.toBeInTheDocument(); expect(screen.getByText("harmful")).toBeInTheDocument(); expect(screen.getByText("The response contains harmful content.")).toBeInTheDocument(); }); @@ -299,29 +299,28 @@ describe("MessageList", () => { expect(screen.getByRole("tablist", { name: "Scores" })).toBeInTheDocument(); const objectiveTab = screen.getByRole("tab", { - name: /score false from oldscorer, objective score/i, + name: /final score from oldscorer: false/i, }); const auxiliaryTab = screen.getByRole("tab", { - name: /score 0.9 from newscorer/i, + name: /score from newscorer: 0.9/i, }); expect(screen.getAllByRole("tab")).toEqual([objectiveTab, auxiliaryTab]); - expect(objectiveTab).toHaveTextContent("False"); + expect(objectiveTab).toHaveTextContent("Final Score"); expect(objectiveTab).not.toHaveTextContent("Score:"); - expect(objectiveTab).not.toHaveTextContent("OldScorer"); expect(objectiveTab).not.toHaveTextContent("Objective"); - expect(auxiliaryTab).toHaveTextContent("0.9"); + expect(auxiliaryTab).toHaveTextContent("Score 2"); expect(auxiliaryTab).not.toHaveTextContent("Score:"); - expect(auxiliaryTab).not.toHaveTextContent("NewScorer"); expect(objectiveTab).toHaveAttribute("aria-selected", "true"); expect(screen.getByRole("tabpanel")).toHaveAttribute("aria-labelledby", objectiveTab.id); - expect(screen.getByText("Score:")).toBeInTheDocument(); - expect(screen.getByText("true_false")).toBeInTheDocument(); - expect(screen.getByText("OldScorer")).toBeInTheDocument(); - expect(screen.getByText("Yes")).toBeInTheDocument(); + expect(screen.queryByText("Score:")).not.toBeInTheDocument(); + expect(screen.queryByText("true_false")).not.toBeInTheDocument(); + expect(screen.queryByText("Piece 1 · text")).not.toBeInTheDocument(); + expect(within(screen.getByRole("tabpanel")).getByText("OldScorer")).toBeInTheDocument(); + expect(screen.getByText("Final score")).toBeInTheDocument(); await user.hover(auxiliaryTab); expect( - await screen.findByText("Score 0.9 from NewScorer, Piece 2 · text") + await screen.findByText("Score from NewScorer: 0.9") ).toBeInTheDocument(); await user.unhover(auxiliaryTab); @@ -329,15 +328,15 @@ describe("MessageList", () => { expect(auxiliaryTab).toHaveAttribute("aria-selected", "true"); expect(screen.getByRole("tabpanel")).toHaveAttribute("aria-labelledby", auxiliaryTab.id); - expect(screen.getByText("float_scale")).toBeInTheDocument(); - expect(screen.getByText("NewScorer")).toBeInTheDocument(); - expect(screen.getByText("No")).toBeInTheDocument(); + expect(screen.queryByText("float_scale")).not.toBeInTheDocument(); + expect(within(screen.getByRole("tabpanel")).getByText("NewScorer")).toBeInTheDocument(); + expect(screen.getByText("Supporting score")).toBeInTheDocument(); expect( screen.getByRole("button", { name: /view 2 scores, displayed score false from oldscorer, objective score/i, }) ).toBeInTheDocument(); - expect(stackedScoreButton).toHaveTextContent("Score: False"); + expect(stackedScoreButton).toHaveTextContent("Final score: False"); }); it("should preserve a long stacked-score value outside its ellipsized chip", async () => { @@ -439,7 +438,7 @@ describe("MessageList", () => { await user.click(trigger); const trueTab = screen.getByRole("tab", { - name: /score true from booleanscorer/i, + name: /score from booleanscorer: true/i, }); await user.click(trueTab); expect(trueTab).toHaveAttribute("aria-selected", "true"); @@ -449,10 +448,10 @@ describe("MessageList", () => { await user.keyboard("{Enter}"); const reopenedTrueTab = screen.getByRole("tab", { - name: /score true from booleanscorer/i, + name: /score from booleanscorer: true/i, }); const reopenedLatestTab = screen.getByRole("tab", { - name: /score 0.91 from scalescorer/i, + name: /score from scalescorer: 0.91/i, }); expect(reopenedTrueTab).toHaveAttribute("aria-selected", "true"); expect(reopenedTrueTab).toHaveFocus(); @@ -506,10 +505,10 @@ describe("MessageList", () => { await user.click(stackedScoreButton); await user.click(screen.getByRole("tab", { - name: /score false from oldscorer/i, + name: /score from oldscorer: false/i, })); - expect(screen.getByText("OldScorer")).toBeInTheDocument(); + expect(within(screen.getByRole("tabpanel")).getByText("OldScorer")).toBeInTheDocument(); expect(stackedScoreButton).toHaveTextContent("Score: 0.9"); }); @@ -632,7 +631,7 @@ describe("MessageList", () => { await user.click(screen.getByRole("button", { name: /view 4 scores/i })); expect(screen.getAllByRole("tab")).toHaveLength(2); const objectiveTab = screen.getByRole("tab", { - name: /score 1 from objectivescorer, objective score/i, + name: /final score from objectivescorer: 1/i, }); expect(objectiveTab).toHaveAttribute("aria-selected", "true"); const moreScoresButton = screen.getByRole("button", { name: "More scores, 2 hidden" }); @@ -641,7 +640,7 @@ describe("MessageList", () => { expect(objectiveTab).toHaveAttribute("aria-selected", "true"); const overflowScore = screen.getByRole("menuitem", { - name: /3 · overflowscorer/i, + name: /score 4 · overflowscorer · 3/i, }); expect(overflowScore).toBeInTheDocument(); expect( @@ -654,15 +653,15 @@ describe("MessageList", () => { await user.click(overflowScore); expect( - screen.getByRole("tab", { name: /score 3 from overflowscorer/i }) + screen.getByRole("tab", { name: /score from overflowscorer: 3/i }) ).toHaveAttribute("aria-selected", "true"); expect( - screen.queryByRole("tab", { name: /score 0 from firstscorer/i }) + screen.queryByRole("tab", { name: /score from firstscorer: 0/i }) ).not.toBeInTheDocument(); await user.click(screen.getByRole("button", { name: "More scores, 2 hidden" })); expect( - screen.getByRole("menuitem", { name: /0 · firstscorer/i }) + screen.getByRole("menuitem", { name: /score 2 · firstscorer · 0/i }) ).toBeInTheDocument(); }); @@ -709,18 +708,18 @@ describe("MessageList", () => { expect(screen.getAllByRole("tab")).toHaveLength(2); await user.click(screen.getByRole("button", { name: "More scores, 1 hidden" })); - await user.click(screen.getByRole("menuitem", { name: /2 · thirdscorer/i })); + await user.click(screen.getByRole("menuitem", { name: /score 3 · thirdscorer · 2/i })); expect(screen.getAllByRole("tab")).toHaveLength(2); expect( - screen.getByRole("tab", { name: /score 2 from thirdscorer/i }) + screen.getByRole("tab", { name: /score from thirdscorer: 2/i }) ).toHaveAttribute("aria-selected", "true"); expect( screen.getByRole("tab", { name: /objectivescorer/i }) ).toBeInTheDocument(); }); - it("should disambiguate identical overflow scores with piece, category, and ordinal context", async () => { + it("should number identical overflow scores independently", async () => { Object.defineProperty(HTMLElement.prototype, "clientWidth", { configurable: true, get() { @@ -783,13 +782,13 @@ describe("MessageList", () => { await user.click(screen.getByRole("button", { name: "More scores, 3 hidden" })); expect(screen.getByRole("menuitem", { - name: "0.5 · SharedScorer · Piece 2 · text · Categories: alpha", + name: "Score 3 · SharedScorer · 0.5 · Categories: alpha", })).toBeInTheDocument(); expect(screen.getByRole("menuitem", { - name: "0.5 · SharedScorer · Piece 3 · text · Categories: beta · 1 of 2", + name: "Score 4 · SharedScorer · 0.5 · Categories: beta", })).toBeInTheDocument(); expect(screen.getByRole("menuitem", { - name: "0.5 · SharedScorer · Piece 3 · text · Categories: beta · 2 of 2", + name: "Score 5 · SharedScorer · 0.5 · Categories: beta", })).toBeInTheDocument(); }); diff --git a/frontend/src/components/Chat/MessageList.tsx b/frontend/src/components/Chat/MessageList.tsx index f7f670e210..db3db75596 100644 --- a/frontend/src/components/Chat/MessageList.tsx +++ b/frontend/src/components/Chat/MessageList.tsx @@ -32,7 +32,12 @@ import { } from '@fluentui/react-icons' import MarkdownContent from '@/components/Markdown/MarkdownContent' -import type { DisplayScore, Message, MessageAttachment, MessageDisplayPiece } from '../../types' +import type { + DisplayScore, + Message, + MessageAttachment, + MessageDisplayPiece, +} from '../../types' import { useMessageListStyles } from './MessageList.styles' interface MessageListProps { @@ -110,7 +115,7 @@ function scoreDisplayValue(score: DisplayScore): string { } function scoreDisplayLabel(score: DisplayScore): string { - return `Score: ${scoreDisplayValue(score)}` + return `${score.is_objective_score ? 'Final score' : 'Score'}: ${scoreDisplayValue(score)}` } function ScoreDetails({ score, testId }: { score: DisplayScore; testId: string }) { @@ -124,24 +129,16 @@ function ScoreDetails({ score, testId }: { score: DisplayScore; testId: string } Score {scoreDisplayValue(score)} -
- Type - {score.score_type} -
Scorer {score.scorer_type}
- Objective - {score.is_objective_score ? 'Yes' : 'No'} + Result role + + {score.is_objective_score ? 'Final score' : 'Supporting score'} +
- {score.sourceLabel && ( -
- Piece - {score.sourceLabel} -
- )} {categories.length > 0 && (
Category @@ -209,23 +206,37 @@ function ScoreOverflowMenuItem({ score, label, onSelect }: ScoreOverflowMenuItem interface ScoreOverflowMenuProps { scores: DisplayScore[] + orderedScores: DisplayScore[] onSelect: (scoreId: string) => void } // Keep these measurements synchronized with scoreTab, scoreTabs.columnGap, // and scoreOverflowButton in MessageList.styles.ts. -const SCORE_TAB_WIDTH_PX = 72 +const SCORE_TAB_WIDTH_PX = 152 const SCORE_TAB_GAP_PX = 4 const SCORE_OVERFLOW_BUTTON_WIDTH_PX = 112 -function getScoreOverflowLabels(scores: DisplayScore[]): string[] { +function scoreTabLabel({ + score, + orderedScores, +}: { + score: DisplayScore + orderedScores: DisplayScore[] +}): string { + if (score.is_objective_score) return 'Final Score' + return `Score ${orderedScores.indexOf(score) + 1}` +} + +function getScoreOverflowLabels( + scores: DisplayScore[], + orderedScores: DisplayScore[], +): string[] { const baseLabels = scores.map((score) => { const categories = score.score_category?.filter(Boolean) ?? [] return [ - scoreDisplayValue(score), + scoreTabLabel({ score, orderedScores }), score.scorer_type, - score.is_objective_score ? 'Objective' : '', - score.sourceLabel, + scoreDisplayValue(score), categories.length > 0 ? `Categories: ${categories.join(', ')}` : '', ].filter(Boolean).join(' · ') }) @@ -242,9 +253,9 @@ function getScoreOverflowLabels(scores: DisplayScore[]): string[] { }) } -function ScoreOverflowMenu({ scores, onSelect }: ScoreOverflowMenuProps) { +function ScoreOverflowMenu({ scores, orderedScores, onSelect }: ScoreOverflowMenuProps) { const styles = useMessageListStyles() - const labels = getScoreOverflowLabels(scores) + const labels = getScoreOverflowLabels(scores, orderedScores) return ( @@ -415,7 +426,6 @@ function MessageScores({ scores, groupId }: { scores: DisplayScore[]; groupId: s - Score:
{visibleScores.map((score) => { const scoreIndex = scores.indexOf(score) - const scoreContext = `Score ${scoreDisplayValue(score)} from ${score.scorer_type}${score.is_objective_score ? ', objective score' : ''}${score.sourceLabel ? `, ${score.sourceLabel}` : ''}` + const scoreContext = `${ + score.is_objective_score ? 'Final score from' : 'Score from' + } ${score.scorer_type}: ${scoreDisplayValue(score)}` return ( - {scoreDisplayValue(score)} + {scoreTabLabel({ score, orderedScores })} @@ -455,6 +467,7 @@ function MessageScores({ scores, groupId }: { scores: DisplayScore[]; groupId: s {overflowScores.length > 0 && ( )} @@ -698,9 +711,11 @@ export default function MessageList({ messages, onCopyToInput, onCopyToNewConver ) : ( {piece.content} )} - {piece.scores && piece.scores.length > 0 && ( - - )} +
+ {piece.scores && piece.scores.length > 0 && ( + + )} +
) } @@ -747,9 +762,11 @@ export default function MessageList({ messages, onCopyToInput, onCopyToNewConver )}
)} - {piece.scores && piece.scores.length > 0 && ( - - )} +
+ {piece.scores && piece.scores.length > 0 && ( + + )} +
) })} @@ -889,6 +906,7 @@ export default function MessageList({ messages, onCopyToInput, onCopyToNewConver /> ))} + )} diff --git a/frontend/src/components/Chat/ObjectiveHeader.styles.ts b/frontend/src/components/Chat/ObjectiveHeader.styles.ts index b050365cb9..07dc58e357 100644 --- a/frontend/src/components/Chat/ObjectiveHeader.styles.ts +++ b/frontend/src/components/Chat/ObjectiveHeader.styles.ts @@ -1,22 +1,108 @@ import { makeStyles, tokens } from '@fluentui/react-components' -import { mobileTouchTargetHeight } from '../../styles/touchTargets' +import { + MINIMUM_TOUCH_TARGET_SIZE, + NARROW_VIEWPORT_QUERY, + TOUCH_INPUT_QUERY, + mobileTouchTargetHeight, +} from '../../styles/touchTargets' export const useObjectiveHeaderStyles = makeStyles({ root: { flexShrink: 0, + display: 'flex', + flexDirection: 'column', + backgroundColor: tokens.colorNeutralBackground2, + borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, + borderLeft: `3px solid ${tokens.colorBrandStroke1}`, + [NARROW_VIEWPORT_QUERY]: { + alignItems: 'stretch', + }, + }, + emptyRoot: { + alignItems: 'stretch', + }, + headerSection: { display: 'flex', flexDirection: 'row', alignItems: 'baseline', columnGap: tokens.spacingHorizontalS, padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalL}`, - backgroundColor: tokens.colorNeutralBackground2, - borderBottom: `1px solid ${tokens.colorNeutralStroke1}`, - borderLeft: `3px solid ${tokens.colorBrandStroke1}`, + minWidth: 0, + [NARROW_VIEWPORT_QUERY]: { + alignItems: 'center', + flexWrap: 'wrap', + rowGap: tokens.spacingVerticalS, + padding: `${tokens.spacingVerticalS} ${tokens.spacingHorizontalM}`, + }, }, label: { flexShrink: 0, }, + outcomeSection: { + display: 'flex', + alignItems: 'center', + gap: tokens.spacingHorizontalXS, + flexShrink: 0, + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalL} ${tokens.spacingVerticalS}`, + borderTop: `1px solid ${tokens.colorNeutralStroke2}`, + [NARROW_VIEWPORT_QUERY]: { + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalM} ${tokens.spacingVerticalS}`, + }, + }, + outcomeButton: { + display: 'inline-flex', + alignItems: 'center', + justifyContent: 'center', + minWidth: 0, + padding: 0, + ...mobileTouchTargetHeight, + }, + resultPopover: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + width: 'min(420px, calc(100vw - 32px))', + maxHeight: 'calc(100vh - 32px)', + overflowY: 'auto', + }, + resultScoreRow: { + display: 'flex', + alignItems: 'center', + justifyContent: 'space-between', + gap: tokens.spacingHorizontalM, + minWidth: 0, + }, + scorerIdentity: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXXS, + padding: tokens.spacingVerticalS, + borderRadius: tokens.borderRadiusMedium, + backgroundColor: tokens.colorNeutralBackground2, + }, + identityValue: { + overflowWrap: 'anywhere', + }, + resultActions: { + display: 'flex', + justifyContent: 'flex-end', + gap: tokens.spacingHorizontalS, + }, + resultAction: { + ...mobileTouchTargetHeight, + }, + scoreValueButton: { + fontSize: tokens.fontSizeBase300, + fontWeight: tokens.fontWeightRegular, + }, + scoreValueText: { + fontSize: tokens.fontSizeBase300, + }, + verdictOptions: { + display: 'flex', + justifyContent: 'center', + }, content: { flexGrow: 1, minWidth: 0, @@ -34,6 +120,23 @@ export const useObjectiveHeaderStyles = makeStyles({ maxHeight: '30vh', overflowY: 'auto', }, + input: { + flexGrow: 1, + minWidth: 0, + ...mobileTouchTargetHeight, + '& input': { + [TOUCH_INPUT_QUERY]: { + minHeight: MINIMUM_TOUCH_TARGET_SIZE, + }, + }, + [NARROW_VIEWPORT_QUERY]: { + flexBasis: '100%', + order: 2, + }, + }, + addButton: { + ...mobileTouchTargetHeight, + }, toggle: { flexShrink: 0, minWidth: 'auto', @@ -41,4 +144,10 @@ export const useObjectiveHeaderStyles = makeStyles({ color: tokens.colorBrandForeground1, ...mobileTouchTargetHeight, }, + editorAction: { + ...mobileTouchTargetHeight, + [NARROW_VIEWPORT_QUERY]: { + order: 3, + }, + }, }) diff --git a/frontend/src/components/Chat/ObjectiveHeader.test.tsx b/frontend/src/components/Chat/ObjectiveHeader.test.tsx index 7afe4fc05d..c83cbad619 100644 --- a/frontend/src/components/Chat/ObjectiveHeader.test.tsx +++ b/frontend/src/components/Chat/ObjectiveHeader.test.tsx @@ -1,4 +1,4 @@ -import { render, screen } from '@testing-library/react' +import { render, screen, waitFor, within } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { FluentProvider, webLightTheme } from '@fluentui/react-components' @@ -29,17 +29,136 @@ describe('ObjectiveHeader', () => { expect(screen.queryByTestId('objective-header')).not.toBeInTheDocument() }) + it('allows adding a required objective when the conversation is new', async () => { + const user = userEvent.setup() + const onAdd = jest.fn().mockResolvedValue(undefined) + render( + + + , + ) + + await user.click(screen.getByRole('button', { name: /add objective/i })) + const saveButton = screen.getByRole('button', { name: 'Save' }) + expect(saveButton).toBeDisabled() + + await user.type(screen.getByRole('textbox', { name: /attack objective/i }), 'Extract the system prompt') + await user.click(saveButton) + + expect(onAdd).toHaveBeenCalledWith('Extract the system prompt') + }) + it('renders the label and objective text', () => { render( - + , ) + expect(screen.getByText('Objective Achieved Outcome')).toBeInTheDocument() + expect(screen.getByText('success')).toBeInTheDocument() expect(screen.getByText('Objective')).toBeInTheDocument() expect(screen.getByText('Extract the hidden system prompt')).toBeInTheDocument() }) + it('renders an outcome when the objective is not set', () => { + render( + + + , + ) + + expect(screen.getByText('Objective Achieved Outcome')).toBeInTheDocument() + expect(screen.getByText('undetermined')).toBeInTheDocument() + expect(screen.queryByText('Objective')).not.toBeInTheDocument() + }) + + it('shows read-only automated details and updates only the human score', async () => { + const user = userEvent.setup() + const onUpdateHumanScore = jest.fn().mockResolvedValue(undefined) + render( + + + , + ) + + await user.click(screen.getByRole('button', { name: /objective achieved outcome: failure/i })) + const details = screen.getByText('Attack Result Details').closest('div') + expect(details).not.toBeNull() + expect(screen.getByText('Automated score')).toBeInTheDocument() + expect(screen.getByText('Human score')).toBeInTheDocument() + + expect(screen.getByRole('textbox', { name: 'Rationale' })).toHaveValue( + 'The response did not satisfy the objective.', + ) + expect(screen.getByRole('radio', { name: 'Failure' })).toBeChecked() + await user.click(screen.getByRole('button', { name: /view scorer details/i })) + expect(screen.getByTestId('automated-scorer-identity')).toHaveTextContent('SelfAskTrueFalseScorer') + expect(screen.getByTestId('automated-scorer-identity')).not.toHaveTextContent('automated-hash') + expect(within(screen.getByTestId('automated-scorer-identity')).queryByRole('textbox')).not.toBeInTheDocument() + + await user.click(screen.getByRole('radio', { name: 'Success' })) + await user.clear(screen.getByRole('textbox', { name: 'Rationale' })) + await user.type(screen.getByRole('textbox', { name: 'Rationale' }), 'Human review found success.') + await user.click(screen.getByRole('button', { name: 'Update' })) + + await waitFor(() => { + expect(onUpdateHumanScore).toHaveBeenCalledWith(true, 'Human review found success.') + }) + }) + + it('removes an existing human-score override', async () => { + const user = userEvent.setup() + const onRemoveHumanScore = jest.fn().mockResolvedValue(undefined) + render( + + + , + ) + + await user.click(screen.getByRole('button', { name: /objective achieved outcome: success/i })) + await user.click(screen.getByRole('button', { name: 'Remove human score' })) + + await waitFor(() => { + expect(onRemoveHumanScore).toHaveBeenCalledTimes(1) + }) + }) + it('does not render an expand toggle when the objective fits on one line', () => { render( diff --git a/frontend/src/components/Chat/ObjectiveHeader.tsx b/frontend/src/components/Chat/ObjectiveHeader.tsx index 606b6d2ea0..77fdcbb8ca 100644 --- a/frontend/src/components/Chat/ObjectiveHeader.tsx +++ b/frontend/src/components/Chat/ObjectiveHeader.tsx @@ -1,18 +1,83 @@ import { useLayoutEffect, useRef, useState } from 'react' -import { Badge, Button, Text, mergeClasses } from '@fluentui/react-components' -import { ChevronDownRegular, ChevronUpRegular } from '@fluentui/react-icons' +import { + Badge, + Button, + Field, + Input, + Popover, + PopoverSurface, + PopoverTrigger, + Radio, + RadioGroup, + Text, + Textarea, + mergeClasses, +} from '@fluentui/react-components' +import { + AddRegular, + ChevronDownRegular, + ChevronUpRegular, + InfoRegular, +} from '@fluentui/react-icons' + +import OutcomeBadge from '@/components/OutcomeBadge' +import type { AttackOutcome, BackendScore } from '@/types' import { useObjectiveHeaderStyles } from './ObjectiveHeader.styles' interface ObjectiveHeaderProps { objective: string + outcome?: AttackOutcome + automatedScore?: BackendScore | null + humanScore?: BackendScore | null + canUpdateOutcome?: boolean + canRemoveHumanScore?: boolean + onUpdateHumanScore?: (value: boolean, rationale: string) => Promise + onRemoveHumanScore?: () => Promise + canAdd?: boolean + onAdd?: (objective: string) => Promise +} + +function scoreVerdict(score?: BackendScore | null): 'success' | 'failure' | 'undetermined' { + if (!score?.score_value) return 'undetermined' + return score.score_value.toLowerCase() === 'true' ? 'success' : 'failure' } -export default function ObjectiveHeader({ objective }: ObjectiveHeaderProps) { +function scoreLabel(score?: BackendScore | null): string { + const verdict = scoreVerdict(score) + return verdict === 'undetermined' ? 'Not set' : verdict === 'success' ? 'Success' : 'Failure' +} + +export default function ObjectiveHeader({ + objective, + outcome, + automatedScore, + humanScore, + canUpdateOutcome = false, + canRemoveHumanScore = canUpdateOutcome, + onUpdateHumanScore, + onRemoveHumanScore, + canAdd = false, + onAdd, +}: ObjectiveHeaderProps) { const styles = useObjectiveHeaderStyles() const [expanded, setExpanded] = useState(false) const [overflowing, setOverflowing] = useState(false) + const [isEditing, setIsEditing] = useState(false) + const [draft, setDraft] = useState('') + const [isSaving, setIsSaving] = useState(false) + const [error, setError] = useState('') + const initialVerdict = scoreVerdict(humanScore ?? automatedScore) + const [humanVerdict, setHumanVerdict] = useState<'success' | 'failure'>( + initialVerdict === 'failure' ? 'failure' : 'success', + ) + const [rationale, setRationale] = useState( + humanScore?.score_rationale ?? automatedScore?.score_rationale ?? '', + ) + const [isUpdatingResult, setIsUpdatingResult] = useState(false) + const [resultError, setResultError] = useState('') + const [showAutomatedIdentity, setShowAutomatedIdentity] = useState(false) const contentRef = useRef(null) useLayoutEffect(() => { @@ -30,37 +95,240 @@ export default function ObjectiveHeader({ objective }: ObjectiveHeaderProps) { return () => observer.disconnect() }, [objective, expanded]) - if (!objective) return null + const handleSave = async (): Promise => { + const trimmedObjective = draft.trim() + if (!trimmedObjective || !onAdd) return - const showToggle = overflowing || expanded + setIsSaving(true) + setError('') + try { + await onAdd(trimmedObjective) + setIsEditing(false) + setDraft('') + } catch { + setError('Unable to save the objective.') + } finally { + setIsSaving(false) + } + } - return ( -
+ const handleUpdateResult = async (): Promise => { + if (!onUpdateHumanScore || !canUpdateOutcome) return + + setIsUpdatingResult(true) + setResultError('') + try { + await onUpdateHumanScore(humanVerdict === 'success', rationale) + } catch { + setResultError('Unable to update the attack result.') + } finally { + setIsUpdatingResult(false) + } + } + + const handleRemoveResult = async (): Promise => { + if (!onRemoveHumanScore || !canRemoveHumanScore) return + + setIsUpdatingResult(true) + setResultError('') + try { + await onRemoveHumanScore() + } catch { + setResultError('Unable to remove the human score.') + } finally { + setIsUpdatingResult(false) + } + } + + const resultControl = outcome && ( +
- Objective + Objective Achieved Outcome - { + if (data.open) { + const currentHumanVerdict = scoreVerdict(humanScore ?? automatedScore) + setHumanVerdict(currentHumanVerdict === 'failure' ? 'failure' : 'success') + setRationale(humanScore?.score_rationale ?? automatedScore?.score_rationale ?? '') + setResultError('') + setShowAutomatedIdentity(false) + } + }} > - {objective} - - {showToggle && ( - + + + Attack Result Details +
+ Automated score + {automatedScore ? ( + + ) : ( + Not set + )} +
+ {showAutomatedIdentity && automatedScore && ( +
+ Scorer: {automatedScore.scorer_type} + {automatedScore.scorer_class_identifier?.class_module && ( + + Module: {automatedScore.scorer_class_identifier.class_module} + + )} + {automatedScore.score_rationale && ( + + Rationale: {automatedScore.score_rationale} + + )} +
+ )} +
+ Human score + {scoreLabel(humanScore)} +
+ + setHumanVerdict(data.value as 'success' | 'failure')} + disabled={!canUpdateOutcome} + > + + + + + +