From 8a197aa73625bfa07e177f6f3596786bb3c9e83f Mon Sep 17 00:00:00 2001 From: Behnam Ousat Date: Thu, 3 Sep 2026 16:30:19 -0700 Subject: [PATCH 1/4] Add manual message scoring Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 097eaf76-ae49-44f8-b44d-9841fa2cc1cb --- .../src/components/Chat/ChatWindow.test.tsx | 3 + frontend/src/components/Chat/ChatWindow.tsx | 14 +- .../src/components/Chat/MessageList.styles.ts | 23 ++++ .../src/components/Chat/MessageList.test.tsx | 39 +++++- frontend/src/components/Chat/MessageList.tsx | 129 +++++++++++++++++- frontend/src/services/api.ts | 9 ++ frontend/src/types/index.ts | 6 + pyrit/backend/main.py | 2 + pyrit/backend/models/scores.py | 16 +++ pyrit/backend/routes/scores.py | 48 +++++++ pyrit/score/__init__.py | 2 + pyrit/score/float_scale/manual_scorer.py | 73 ++++++++++ tests/unit/backend/test_api_routes.py | 71 +++++++++- tests/unit/score/test_manual_scorer.py | 44 ++++++ 14 files changed, 469 insertions(+), 10 deletions(-) create mode 100644 pyrit/backend/models/scores.py create mode 100644 pyrit/backend/routes/scores.py create mode 100644 pyrit/score/float_scale/manual_scorer.py create mode 100644 tests/unit/score/test_manual_scorer.py diff --git a/frontend/src/components/Chat/ChatWindow.test.tsx b/frontend/src/components/Chat/ChatWindow.test.tsx index b932a1e230..5ba7f8ec82 100644 --- a/frontend/src/components/Chat/ChatWindow.test.tsx +++ b/frontend/src/components/Chat/ChatWindow.test.tsx @@ -35,6 +35,9 @@ jest.mock("../../services/api", () => ({ createConversation: jest.fn(), changeMainConversation: jest.fn(), }, + scoresApi: { + createManualScore: jest.fn(), + }, convertersApi: { listConverterCatalog: jest.fn(), listConverters: jest.fn(), diff --git a/frontend/src/components/Chat/ChatWindow.tsx b/frontend/src/components/Chat/ChatWindow.tsx index bc7286d86e..a894f6601e 100644 --- a/frontend/src/components/Chat/ChatWindow.tsx +++ b/frontend/src/components/Chat/ChatWindow.tsx @@ -33,7 +33,7 @@ import type { PieceConversion } from './converterTypes' import { PIECE_TYPE_TO_DATA_TYPE, basenameFromValue, buildMediaUrl, dataTypeToAttachmentKind, isPathDataType } from './converterTypes' import LabelsBar from '../Labels/LabelsBar' import type { ChatInputAreaHandle } from './ChatInputArea' -import { attacksApi } from '../../services/api' +import { attacksApi, scoresApi } from '../../services/api' import { toApiError } from '../../services/errors' import { buildMessagePieces, backendMessagesToFrontend } from '../../utils/messageMapper' import { exportConversation } from '../../utils/conversationExport' @@ -690,6 +690,17 @@ export default function ChatWindow({ isMutationLocked, ]) + const handleManualScore = useCallback(async (messageId: string, value: number, rationale: string) => { + if (!attackResultId || !activeConversationId || isMutationLocked) return + + await scoresApi.createManualScore({ + message_id: messageId, + value, + rationale, + }) + await loadConversation(attackResultId, activeConversationId) + }, [activeConversationId, attackResultId, isMutationLocked, loadConversation]) + const singleTurnLimitReached = activeTarget?.capabilities?.supports_multi_turn === false && messages.some(m => m.role === 'user') // "Continue with your target" — clone the current conversation into a new attack @@ -883,6 +894,7 @@ export default function ChatWindow({ isCrossTarget={isCrossTargetLocked || isTargetResolutionLocked} noTargetSelected={!activeTarget} globalMarkdown={globalMarkdown} + onManualScore={!isMutationLocked && attackResultId ? handleManualScore : undefined} /> { expect(screen.getByText("Assistant message test")).toBeInTheDocument(); }); + it("should submit a manual score for a persisted message piece", async () => { + const user = userEvent.setup(); + const onManualScore = jest.fn().mockResolvedValue(undefined); + const messages: Message[] = [ + { + role: "assistant", + content: "Response to score", + timestamp: new Date().toISOString(), + displayPieces: [ + { + type: "text", + pieceId: "piece-manual", + pieceIndex: 0, + content: "Response to score", + scores: [], + }, + ], + }, + ]; + + render( + + + + ); + + await user.click(screen.getByRole("button", { name: "Add manual score" })); + await user.click(screen.getByRole("button", { name: "0.33" })); + await user.type(screen.getByLabelText("Manual score rationale"), "Partially satisfied"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onManualScore).toHaveBeenCalledWith("piece-manual", 0.33, "Partially satisfied"); + }); + expect(screen.queryByText("Manual score")).not.toBeInTheDocument(); + }); + it("should show the message score and its details when present", async () => { const user = userEvent.setup(); const scoredMessages: Message[] = [ diff --git a/frontend/src/components/Chat/MessageList.tsx b/frontend/src/components/Chat/MessageList.tsx index f7f670e210..1295a58ef8 100644 --- a/frontend/src/components/Chat/MessageList.tsx +++ b/frontend/src/components/Chat/MessageList.tsx @@ -7,6 +7,8 @@ import { MessageBarBody, Button, Badge, + Field, + Input, Menu, MenuItem, MenuList, @@ -19,6 +21,7 @@ import { TabList, Tooltip, Spinner, + Textarea, mergeClasses, } from '@fluentui/react-components' import { @@ -27,6 +30,7 @@ import { ArrowReplyRegular, BranchForkRegular, ChatAddRegular, + GaugeRegular, MoreHorizontalRegular, OpenRegular, } from '@fluentui/react-icons' @@ -57,6 +61,8 @@ interface MessageListProps { noTargetSelected?: boolean /** Conversation-wide default: render message text as Markdown. */ globalMarkdown?: boolean + /** Persist a manual score for a message piece. */ + onManualScore?: (messageId: string, value: number, rationale: string) => Promise } /** Image that shows a spinner while loading. */ @@ -113,6 +119,105 @@ function scoreDisplayLabel(score: DisplayScore): string { return `Score: ${scoreDisplayValue(score)}` } +const MANUAL_SCORE_PRESETS = [0, 0.25, 0.33, 0.5, 0.66, 0.75, 1] + +interface ManualScorePopoverProps { + messageId: string + onSave: (messageId: string, value: number, rationale: string) => Promise +} + +function ManualScorePopover({ messageId, onSave }: ManualScorePopoverProps) { + const styles = useMessageListStyles() + const [isOpen, setIsOpen] = useState(false) + const [value, setValue] = useState('0.5') + const [rationale, setRationale] = useState('') + const [isSaving, setIsSaving] = useState(false) + const [error, setError] = useState('') + const numericValue = Number(value) + const isValueValid = value.trim() !== '' && Number.isFinite(numericValue) && numericValue >= 0 && numericValue <= 1 + + const handleSave = async () => { + if (!isValueValid) return + + setIsSaving(true) + setError('') + try { + await onSave(messageId, numericValue, rationale) + setIsOpen(false) + setRationale('') + } catch { + setError('Unable to save the manual score.') + } finally { + setIsSaving(false) + } + } + + return ( + { + setIsOpen(data.open) + if (!data.open) setError('') + }} + > + + + + ))} + + + setValue(data.value)} + aria-label="Manual score value" + /> + + +