feat(prompts): add table to compare metrics across prompt versions, [promptName]/metrics

packages/shared/prisma/schema.prisma

@@ -276,6 +276,8 @@ view TraceView {
 }
 
 // Update ObservationView below when making changes to this model!
+// traceId is optional only due to timing during data injestion 
+// (traceId is not necessarily known at the time of observation creation)
 model Observation {
     id                  String           @id @default(cuid())
     traceId             String?          @map("trace_id")

web/src/features/prompts/components/prompt-detail.tsx

@@ -9,6 +9,7 @@ import {
   ChatMlArraySchema,
   OpenAiMessageView,
 } from "@/src/components/trace/IOPreview";
+import { Tabs, TabsList, TabsTrigger } from "@/src/components/ui/tabs";
 import { Badge } from "@/src/components/ui/badge";
 import { Button } from "@/src/components/ui/button";
 import { CodeView, JSONView } from "@/src/components/ui/CodeJsonViewer";
@@ -151,6 +152,18 @@ export const PromptDetail = () => {
                   path={(name) => `/project/${projectId}/prompts/${name}`}
                   listKey="prompts"
                 />
+                <Tabs value="editor">
+                  <TabsList>
+                    <TabsTrigger value="editor">Editor</TabsTrigger>
+                    <TabsTrigger value="table" asChild>
+                      <Link
+                        href={`/project/${projectId}/prompts/${encodeURIComponent(promptName)}/version-table`}
+                      >
+                        Table
+                      </Link>
+                    </TabsTrigger>
+                  </TabsList>
+                </Tabs>
               </>
             }
           />

web/src/features/prompts/server/routers/promptRouter.ts

@@ -507,6 +507,106 @@ export const promptRouter = createTRPCRouter({
       });
       return joinedPromptAndUsers;
     }),
+  metrics: protectedProjectProcedure
+    .input(z.object({ projectId: z.string(), promptIds: z.array(z.string()) }))
+    .query(async ({ input, ctx }) => {
+      throwIfNoAccess({
+        session: ctx.session,
+        projectId: input.projectId,
+        scope: "prompts:read",
+      });
+
+      const metrics = await ctx.prisma.$queryRaw<
+        Array<{
+          id: string;
+          observationCount: number;
+          firstUsed: Date | null;
+          lastUsed: Date | null;
+          averageObservationScore: number | null;
+          medianOutputTokens: number | null;
+          medianInputTokens: number | null;
+          medianTotalCost: number | null;
+          medianLatency: number | null;
+        }>
+      >(
+        Prisma.sql`
+        SELECT 
+          prompt_id AS "id",
+          count(*) AS "observationCount",
+          MIN(start_time) AS "firstUsed",
+          MAX(start_time) AS "lastUsed",
+          AVG(value) AS "averageObservationScore",
+          PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY completion_tokens) AS "medianOutputTokens",
+          PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY prompt_tokens) AS "medianInputTokens",
+          PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY calculated_total_cost) AS "medianTotalCost",
+          PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER BY latency) AS "medianLatency"
+        FROM 
+        (
+          SELECT s.id AS "scoreId", s.value, os.* from scores s 
+          LEFT JOIN LATERAL (
+            SELECT 
+              *
+            FROM 
+              "observations_view" ov
+            WHERE 
+              s.observation_id IS NOT NULL 
+              AND ov.id = s.observation_id 
+              AND "type" = 'GENERATION'
+              AND "project_id" = ${input.projectId}
+          ) AS os ON true 
+              RIGHT JOIN prompts p ON p.id = os.prompt_id
+              WHERE prompt_id IS NOT NULL
+              AND p.id in (${Prisma.join(input.promptIds)})
+              AND os.project_id = ${input.projectId}
+        ) GROUP BY "prompt_id"
+    `,
+      );
+
+      const averageTraceObservationScore = await ctx.prisma.$queryRaw<
+        Array<{
+          id: string;
+          averageTraceObservationScore: number;
+        }>
+      >(
+        Prisma.sql`
+        SELECT
+          os.prompt_id AS "id",
+          AVG(value) AS "averageTraceObservationScore"
+        FROM
+          (
+            SELECT
+              s.id,
+              s.value,
+              ov.trace_id,
+              ov.prompt_id,
+              ROW_NUMBER() OVER (PARTITION BY ov.prompt_id, ov.trace_id ORDER BY s.id) AS row_num
+            FROM
+              scores s
+            LEFT JOIN "observations_view" ov ON ov.trace_id = s.trace_id
+            WHERE
+              s.observation_id IS NULL
+              AND ov."type" = 'GENERATION'
+              AND ov."project_id" = ${input.projectId}
+          ) AS os
+        WHERE
+          os.row_num = 1
+          AND os.prompt_id IN (${Prisma.join(input.promptIds)})
+        GROUP BY os.prompt_id`,
+      );
+
+      const averageTraceScoreMap = averageTraceObservationScore.reduce(
+        (acc, { id, averageTraceObservationScore }) => {
+          acc[id] = averageTraceObservationScore;
+          return acc;
+        },
+        {} as Record<string, number>,
+      );
+
+      return metrics.map((metric) => ({
+        ...metric,
+        averageTraceScore: averageTraceScoreMap[metric.id] ?? null,
+      }));
+    }),
 });
 
 const generatePromptQuery = (