diff --git a/.server-changes/billing-limits.md b/.server-changes/billing-limits.md new file mode 100644 index 0000000000..ba40d03928 --- /dev/null +++ b/.server-changes/billing-limits.md @@ -0,0 +1,24 @@ +--- +area: webapp +type: feature +--- + +Add billing limits. Customers set a spend cap; when usage crosses it, billable +environments pause for a grace period, new triggers are rejected once it ends, +and a recovery flow resumes or cancels the queued backlog. Reconciliation keeps +the webapp converged to billing's state. + +## Manual pause during billing enforcement + +While `pauseSource=BILLING_LIMIT`, manual resume is rejected and manual pause is +a silent no-op (`PauseEnvironmentService` returns success with state `paused`). +We do not stack a manual pause on top of billing enforcement because resolve +converge unpauses all `BILLING_LIMIT`-paused environments for the org. + +API callers that pause during enforcement should expect the environment to +resume when the billing limit is resolved. The queues UI hides pause/resume in +this state; see `manualPauseEnvironmentGuard.server.ts`. + +The admin `runs.enable` endpoint skips billing-paused environments when +re-enabling or disabling org runs (returns them in `skipped`, not `failures` or +the update count). They resume only after the billing limit is resolved. diff --git a/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx b/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx new file mode 100644 index 0000000000..f950fbf170 --- /dev/null +++ b/apps/webapp/app/components/billing/AnimatedOrgBannerBar.tsx @@ -0,0 +1,57 @@ +import { ExclamationCircleIcon } from "@heroicons/react/20/solid"; +import { AnimatePresence, motion } from "framer-motion"; +import tileBgPath from "~/assets/images/error-banner-tile@2x.png"; +import { Icon } from "~/components/primitives/Icon"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { cn } from "~/utils/cn"; + +type AnimatedOrgBannerBarProps = { + show: boolean; + variant: "warning" | "error"; + children: React.ReactNode; + action?: React.ReactNode; +}; + +export function AnimatedOrgBannerBar({ + show, + variant, + children, + action, +}: AnimatedOrgBannerBarProps) { + return ( + + {show ? ( + +
+ + + {children} + +
+ {action} + + ) : null} + + ); +} diff --git a/apps/webapp/app/components/billing/BillingAlertsSection.tsx b/apps/webapp/app/components/billing/BillingAlertsSection.tsx new file mode 100644 index 0000000000..3c30468786 --- /dev/null +++ b/apps/webapp/app/components/billing/BillingAlertsSection.tsx @@ -0,0 +1,389 @@ +import { conform, list, requestIntent, useFieldList, useForm } from "@conform-to/react"; +import { parse } from "@conform-to/zod"; +import { PlusIcon, TrashIcon } from "@heroicons/react/20/solid"; +import { Form, useActionData, useSearchParams } from "@remix-run/react"; +import { Fragment, useEffect, useMemo, useRef, useState } from "react"; +import { z } from "zod"; +import { + emailsMatchSaved, + getAlertPreviewLimitCents, + getBillingLimitMode, + getEffectiveLimitCents, + hasLegacySpikeAlertLevels, + isPercentageAlertMode, + MAX_ABSOLUTE_ALERTS, + MAX_PERCENTAGE_ALERTS, + MAX_PERCENTAGE_THRESHOLD, + previewDollarAmountForPercent, + storedAlertsToThresholds, + thresholdsMatchSaved, + thresholdValuesAreUnique, + type BillingAlertsFormData, +} from "~/components/billing/billingAlertsFormat"; +import { AnimatedCallout } from "~/components/primitives/AnimatedCallout"; +import { Button } from "~/components/primitives/Buttons"; +import { Fieldset } from "~/components/primitives/Fieldset"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { FormError } from "~/components/primitives/FormError"; +import { Header2 } from "~/components/primitives/Headers"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Label } from "~/components/primitives/Label"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { TextLink } from "~/components/primitives/TextLink"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { formatCurrency } from "~/utils/numberFormatter"; +import { docsPath } from "~/utils/pathBuilder"; + +export const billingAlertsSchema = z.object({ + emails: z.preprocess((i) => { + if (typeof i === "string") return [i]; + + if (Array.isArray(i)) { + const emails = i.filter((v) => typeof v === "string" && v !== ""); + if (emails.length === 0) { + return [""]; + } + return emails; + } + + return [""]; + }, z.string().email().array().nonempty("At least one email is required")), + alertLevels: z.preprocess( + (i) => { + const values = typeof i === "string" ? [i] : Array.isArray(i) ? i : []; + return values.filter((v) => v !== "").map((v) => Number(v)); + }, + z.number().array().refine(thresholdValuesAreUnique, "Each alert must be unique"), + ), +}); + +export type { BillingAlertsFormData } from "~/components/billing/billingAlertsFormat"; + +type BillingAlertsActionData = { + formIntent: "billing-alerts"; + submission: ReturnType>; +}; + +type BillingAlertsSectionProps = { + alerts: BillingAlertsFormData; + billingLimit: BillingLimitResult; + planLimitCents: number; + alertsResetRequested?: boolean; +}; + +type ThresholdRow = { + id: number; + value: string; +}; + +function isEmptyThresholdRow(value: string): boolean { + const parsed = Number(value); + return value === "" || !Number.isFinite(parsed) || parsed <= 0; +} + +function toThresholdRows(values: number[]): ThresholdRow[] { + return values.map((value, index) => ({ id: index, value: String(value) })); +} + +function parseThresholdValues(rows: ThresholdRow[]): number[] { + return rows + .map((row) => Number(row.value)) + .filter((value) => Number.isFinite(value) && value > 0); +} + +function isDuplicateThresholdRow(rows: ThresholdRow[], index: number): boolean { + const value = rows[index]?.value; + if (!value || isEmptyThresholdRow(value)) { + return false; + } + + const parsed = Number(value); + return rows.some( + (row, rowIndex) => + rowIndex !== index && !isEmptyThresholdRow(row.value) && Number(row.value) === parsed, + ); +} + +export function BillingAlertsSection({ + alerts, + billingLimit, + planLimitCents, + alertsResetRequested = false, +}: BillingAlertsSectionProps) { + const [searchParams, setSearchParams] = useSearchParams(); + const [showResetBanner, setShowResetBanner] = useState(alertsResetRequested); + + useEffect(() => { + if (!alertsResetRequested) { + return; + } + + setShowResetBanner(true); + + if (searchParams.get("alertsReset") !== "1") { + return; + } + + const next = new URLSearchParams(searchParams); + next.delete("alertsReset"); + setSearchParams(next, { replace: true }); + }, [alertsResetRequested, searchParams, setSearchParams]); + + const billingLimitMode = getBillingLimitMode(billingLimit); + const isPercentageMode = isPercentageAlertMode(billingLimitMode); + const effectiveLimitCents = getEffectiveLimitCents(billingLimit, planLimitCents); + const alertPreviewLimitCents = getAlertPreviewLimitCents( + alerts, + effectiveLimitCents, + planLimitCents, + ); + const maxAlerts = isPercentageMode ? MAX_PERCENTAGE_ALERTS : MAX_ABSOLUTE_ALERTS; + + const savedThresholds = useMemo( + () => storedAlertsToThresholds(alerts, billingLimitMode, effectiveLimitCents, planLimitCents), + [alerts, billingLimitMode, effectiveLimitCents, planLimitCents], + ); + const savedEmails = useMemo(() => alerts.emails, [alerts.emails]); + const hasLegacySpikes = useMemo( + () => hasLegacySpikeAlertLevels(alerts, billingLimitMode, effectiveLimitCents, planLimitCents), + [alerts, billingLimitMode, effectiveLimitCents, planLimitCents], + ); + + const nextThresholdIdRef = useRef(savedThresholds.length); + const [thresholdRows, setThresholdRows] = useState(() => + toThresholdRows(savedThresholds), + ); + const [emailValues, setEmailValues] = useState( + savedEmails.length > 0 ? [...savedEmails, ""] : [""], + ); + const actionData = useActionData(); + const alertsSubmission = + actionData?.formIntent === "billing-alerts" ? actionData.submission : undefined; + + const currentThresholds = parseThresholdValues(thresholdRows); + const isDirty = + hasLegacySpikes || + !thresholdsMatchSaved(currentThresholds, savedThresholds) || + !emailsMatchSaved(emailValues, savedEmails); + const lastSubmission = isDirty ? alertsSubmission : undefined; + + const [form, { emails, alertLevels }] = useForm({ + id: "billing-alerts", + lastSubmission: lastSubmission as any, + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parse(formData, { schema: billingAlertsSchema }); + }, + defaultValue: { + emails: emailValues, + alertLevels: savedThresholds.map(String), + }, + }); + + const emailFields = useFieldList(form.ref, emails); + + useEffect(() => { + nextThresholdIdRef.current = savedThresholds.length; + setThresholdRows(toThresholdRows(savedThresholds)); + setEmailValues(savedEmails.length > 0 ? [...savedEmails, ""] : [""]); + }, [savedThresholds, savedEmails]); + + function updateThresholdRow(index: number, rawValue: string) { + setThresholdRows((current) => + current.map((row, rowIndex) => (rowIndex === index ? { ...row, value: rawValue } : row)), + ); + } + + function removeThreshold(index: number) { + setThresholdRows((current) => current.filter((_, rowIndex) => rowIndex !== index)); + } + + function addThreshold() { + if (thresholdRows.length >= maxAlerts) return; + if (thresholdRows.some((row) => isEmptyThresholdRow(row.value))) return; + + nextThresholdIdRef.current += 1; + setThresholdRows((current) => [...current, { id: nextThresholdIdRef.current, value: "" }]); + } + + const hasEmptyThreshold = thresholdRows.some((row) => isEmptyThresholdRow(row.value)); + const hasDuplicateThresholds = !thresholdValuesAreUnique(currentThresholds); + const canAddThreshold = thresholdRows.length < maxAlerts && !hasEmptyThreshold; + const showAlertsSave = isDirty && !hasDuplicateThresholds; + + return ( +
+
+ Billing alerts + + Receive an email when your compute spend crosses different thresholds. You can also learn + how to{" "} + reduce your compute spend. + +
+
+ +
+ setShowResetBanner(false)} + > + Billing alerts were reset because they no longer match the selected billing limit + configuration. + + +
+ + +
+ {thresholdRows.map((row, index) => { + const parsed = Number(row.value); + const isDuplicate = isDuplicateThresholdRow(thresholdRows, index); + + return ( +
+
+ {isPercentageMode ? ( + <> + updateThresholdRow(index, e.target.value)} + onBlur={(e) => { + if (e.target.value === "") return; + const clamped = Math.min( + MAX_PERCENTAGE_THRESHOLD, + Math.max(1, Number(e.target.value)), + ); + if (String(clamped) !== e.target.value) { + updateThresholdRow(index, String(clamped)); + } + }} + min={1} + max={MAX_PERCENTAGE_THRESHOLD} + step={1} + placeholder="75" + className="w-24" + fullWidth={false} + /> + % + + ( + {formatCurrency( + previewDollarAmountForPercent( + Number.isFinite(parsed) ? parsed : 0, + alertPreviewLimitCents, + ), + false, + )} + ) + + + ) : ( + updateThresholdRow(index, e.target.value)} + min={0.01} + step={0.01} + placeholder="100" + icon={ + + $ + + } + className="max-w-xs pl-px" + fullWidth={false} + /> + )} + +
+ {isDuplicate && This alert threshold is already in use} +
+ ); + })} +
+ + {alertLevels.error} + + {canAddThreshold && ( + + )} +
+ + + + {emailFields.map((email, index) => { + const { defaultValue: _emailDefaultValue, ...emailInputProps } = conform.input( + email, + { type: "email" }, + ); + + return ( + + { + const nextValue = e.target.value; + setEmailValues((current) => { + const next = [...current]; + next[index] = nextValue; + if ( + emailFields.length === next.length && + next.every((value) => value !== "") + ) { + requestIntent(form.ref.current ?? undefined, list.append(emails.name)); + return [...next, ""]; + } + return next; + }); + }} + fullWidth + /> + {email.error} + + ); + })} + + + + Update alerts + + } + /> +
+
+
+ ); +} diff --git a/apps/webapp/app/components/billing/BillingLimitConfigSection.tsx b/apps/webapp/app/components/billing/BillingLimitConfigSection.tsx new file mode 100644 index 0000000000..555de2cadf --- /dev/null +++ b/apps/webapp/app/components/billing/BillingLimitConfigSection.tsx @@ -0,0 +1,335 @@ +import { conform, useForm, type Submission } from "@conform-to/react"; + +import { parse } from "@conform-to/zod"; +import { Form, useActionData } from "@remix-run/react"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { z } from "zod"; +import { AnimatedCallout } from "~/components/primitives/AnimatedCallout"; +import { getBillingLimitMode } from "~/components/billing/billingAlertsFormat"; +import { formatGracePeriodMs } from "~/components/billing/billingLimitFormat"; +import { Button } from "~/components/primitives/Buttons"; +import { CheckboxWithLabel } from "~/components/primitives/Checkbox"; +import { Fieldset } from "~/components/primitives/Fieldset"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { FormError } from "~/components/primitives/FormError"; +import { Header2 } from "~/components/primitives/Headers"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { cn } from "~/utils/cn"; +import { formatCurrency } from "~/utils/numberFormatter"; + +export const billingLimitFormSchema = z.discriminatedUnion("mode", [ + z.object({ + mode: z.literal("none"), + cancelInProgressRuns: z + .preprocess((v) => v === "on" || v === true || v === "true", z.boolean()) + .optional(), + }), + z.object({ + mode: z.literal("plan"), + cancelInProgressRuns: z + .preprocess((v) => v === "on" || v === true || v === "true", z.boolean()) + .optional(), + }), + z.object({ + mode: z.literal("custom"), + amount: z.coerce + .number({ invalid_type_error: "Not a valid amount" }) + .positive("Amount must be greater than 0"), + cancelInProgressRuns: z + .preprocess((v) => v === "on" || v === true || v === "true", z.boolean()) + .optional(), + }), +]); + +type BillingLimitFormValue = z.infer; + +type BillingLimitActionData = { + formIntent: "billing-limit"; + submission: Submission; +}; + +export function isBillingLimitFormDirty(input: { + billingLimit: BillingLimitResult; + mode: "none" | "plan" | "custom"; + customAmount: string; + cancelInProgressRuns: boolean; +}): boolean { + const needsInitialSave = !input.billingLimit.isConfigured; + const savedMode = getBillingLimitMode(input.billingLimit); + const savedCustomAmount = + input.billingLimit.isConfigured && input.billingLimit.mode === "custom" + ? (input.billingLimit.amountCents / 100).toFixed(2) + : ""; + const savedCancelInProgressRuns = + input.billingLimit.isConfigured && input.billingLimit.cancelInProgressRuns; + + const isLimitDirty = + input.mode !== savedMode || + (input.mode === "custom" && input.customAmount !== savedCustomAmount); + + return ( + needsInitialSave || isLimitDirty || input.cancelInProgressRuns !== savedCancelInProgressRuns + ); +} + +export function getBillingLimitFormLastSubmission( + submission: BillingLimitActionData["submission"] | undefined, + mode: "none" | "plan" | "custom", + isDirty: boolean, +) { + if (!isDirty || !submission) { + return undefined; + } + + if (mode !== "custom" && submission.error?.amount) { + const { amount: _amount, ...remainingErrors } = submission.error; + return { + ...submission, + error: remainingErrors, + }; + } + + return submission; +} + +type BillingLimitConfigSectionProps = { + billingLimit: BillingLimitResult; + planLimitCents: number; +}; + +export function BillingLimitConfigSection({ + billingLimit, + planLimitCents, +}: BillingLimitConfigSectionProps) { + const gracePeriodLabel = formatGracePeriodMs(billingLimit.gracePeriodMs); + + const savedMode = getBillingLimitMode(billingLimit); + const savedCustomAmount = + billingLimit.isConfigured && billingLimit.mode === "custom" + ? (billingLimit.amountCents / 100).toFixed(2) + : ""; + const savedCancelInProgressRuns = billingLimit.isConfigured && billingLimit.cancelInProgressRuns; + + const [mode, setMode] = useState<"none" | "plan" | "custom">(savedMode); + const [customAmount, setCustomAmount] = useState(savedCustomAmount); + const [cancelInProgressRuns, setCancelInProgressRuns] = useState(savedCancelInProgressRuns); + const customAmountInputRef = useRef(null); + + useEffect(() => { + setMode(savedMode); + setCustomAmount(savedCustomAmount); + setCancelInProgressRuns(savedCancelInProgressRuns); + }, [savedMode, savedCustomAmount, savedCancelInProgressRuns]); + + function handleModeChange(value: string) { + const nextMode = value as typeof mode; + if (mode === "custom" && nextMode !== "custom") { + setCustomAmount(savedCustomAmount); + } + setMode(nextMode); + if (nextMode === "custom") { + window.setTimeout(() => customAmountInputRef.current?.focus(), 0); + } + } + + const actionData = useActionData(); + const limitSubmission = + actionData?.formIntent === "billing-limit" ? actionData.submission : undefined; + + const needsInitialSave = !billingLimit.isConfigured; + const isLimitDirty = + mode !== savedMode || (mode === "custom" && customAmount !== savedCustomAmount); + const isDirty = isBillingLimitFormDirty({ + billingLimit, + mode, + customAmount, + cancelInProgressRuns, + }); + const lastSubmission = useMemo( + () => getBillingLimitFormLastSubmission(limitSubmission, mode, isDirty), + [limitSubmission, mode, isDirty], + ); + + const [form, fields] = useForm({ + id: "billing-limit", + lastSubmission: lastSubmission as any, + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parse(formData, { schema: billingLimitFormSchema }); + }, + defaultValue: { + mode: savedMode, + }, + }); + + useEffect(() => { + form.ref.current?.dispatchEvent(new Event("input", { bubbles: true })); + }, [customAmount, form.ref, mode]); + + const planLimitLabel = formatCurrency(planLimitCents / 100, false); + const showPlanInfoCallout = mode === "plan"; + const showCustomInfoCallout = mode === "custom"; + const showNoneWarningCallout = mode === "none"; + + return ( +
+
+ Billing limit + + Set a monthly compute spend limit for your organization. When the limit is reached, + billable environments enter a grace period before new triggers are rejected. + +
+ +
+ +
+ + + +
+ {`My plan limit (${planLimitLabel})`} + } + description={`Pause billable environments when monthly spend reaches ${planLimitLabel}.`} + /> +
+ + + +
+
+ +
+
+ + {mode === "custom" && ( + + setCustomAmount(e.target.value)} + placeholder="Custom limit amount" + icon={ + + $ + + } + className="pl-px" + fullWidth + /> + {fields.amount && ( + {fields.amount.error} + )} + + )} +
+
+ + + +
+
+ +
+ +
+ + Without a billing limit, runs will continue even if usage spikes unexpectedly. You + may have to pay higher fees before you notice. + +
+
+ + + {mode !== "none" && ( + + )} + + Save billing limit + + } + /> +
+
+
+ ); +} + +function LimitReachedCalloutContent({ + gracePeriodLabel, + cancelInProgressRuns, +}: { + gracePeriodLabel: string; + cancelInProgressRuns: boolean; +}) { + return ( + <> + When this limit is reached, queued runs will be held for {gracePeriodLabel}, then new triggers + will be rejected until you increase or remove the limit. Limits are enforced with a short + delay, so spend may briefly exceed the limit before grace begins. See our{" "} + + terms + {" "} + for refund policy details. + {cancelInProgressRuns ? ( + <> In-progress runs will be cancelled when the limit is hit. + ) : null} + + ); +} diff --git a/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx b/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx new file mode 100644 index 0000000000..4f634717e5 --- /dev/null +++ b/apps/webapp/app/components/billing/BillingLimitRecoveryPanel.tsx @@ -0,0 +1,240 @@ +import { conform, useForm } from "@conform-to/react"; +import { parse } from "@conform-to/zod"; +import { Form, useActionData, useNavigation } from "@remix-run/react"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { z } from "zod"; +import { Button } from "~/components/primitives/Buttons"; +import { Callout } from "~/components/primitives/Callout"; +import { Fieldset } from "~/components/primitives/Fieldset"; +import { FormButtons } from "~/components/primitives/FormButtons"; +import { FormError } from "~/components/primitives/FormError"; +import { Header2 } from "~/components/primitives/Headers"; +import { Input } from "~/components/primitives/Input"; +import { InputGroup } from "~/components/primitives/InputGroup"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { DateTime } from "~/components/primitives/DateTime"; +import { RadioGroup, RadioGroupItem } from "~/components/primitives/RadioButton"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { formatCurrency } from "~/utils/numberFormatter"; + +export const billingLimitRecoveryFormSchema = z + .object({ + action: z.enum(["increase", "remove"]), + newAmount: z.coerce + .number({ invalid_type_error: "Not a valid amount" }) + .positive("Amount must be greater than 0") + .optional(), + resumeMode: z.enum(["queue", "new_only"]), + }) + .superRefine((value, ctx) => { + if (value.action === "increase" && (value.newAmount === undefined || value.newAmount <= 0)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "Amount must be greater than 0", + path: ["newAmount"], + }); + } + }); + +type BillingLimitRecoveryActionData = { + formIntent: "billing-limit-resolve"; + submission: ReturnType>; +}; + +export function BillingLimitRecoveryPanel({ + billingLimit, + currentSpendCents, + queuedRunCount, + suggestedNewLimitDollars, +}: { + billingLimit: BillingLimitResult & { isConfigured: true }; + currentSpendCents: number; + queuedRunCount: number; + suggestedNewLimitDollars: number; +}) { + const { limitState } = billingLimit; + const isGrace = limitState.status === "grace"; + const isRejected = limitState.status === "rejected"; + + const [action, setAction] = useState<"increase" | "remove">("increase"); + const [newAmount, setNewAmount] = useState(String(suggestedNewLimitDollars)); + const [resumeMode, setResumeMode] = useState<"queue" | "new_only">("queue"); + const newAmountInputRef = useRef(null); + + useEffect(() => { + setNewAmount(String(suggestedNewLimitDollars)); + }, [suggestedNewLimitDollars]); + + function handleActionChange(value: string) { + const nextAction = value as typeof action; + setAction(nextAction); + if (nextAction === "increase") { + window.setTimeout(() => newAmountInputRef.current?.focus(), 0); + } + } + + const actionData = useActionData(); + const recoverySubmission = + actionData?.formIntent === "billing-limit-resolve" ? actionData.submission : undefined; + + const [form, fields] = useForm({ + id: "billing-limit-resolve", + lastSubmission: recoverySubmission as any, + shouldRevalidate: "onInput", + onValidate({ formData }) { + return parse(formData, { schema: billingLimitRecoveryFormSchema }); + }, + defaultValue: { + action: "increase", + newAmount: suggestedNewLimitDollars, + resumeMode: "queue", + }, + }); + + useEffect(() => { + form.ref.current?.dispatchEvent(new Event("input", { bubbles: true })); + }, [action, form.ref, newAmount, resumeMode]); + + const navigation = useNavigation(); + const isSubmitting = navigation.state === "submitting"; + + const queuedRunsLabel = useMemo(() => { + if (queuedRunCount === 0) { + return null; + } + return `~${queuedRunCount.toLocaleString()} run${ + queuedRunCount === 1 ? "" : "s" + } waiting in queue`; + }, [queuedRunCount]); + + return ( +
+
+ Action required + + + {isGrace ? ( + <> + Your organization has reached its billing limit. Processing is paused and new runs + are queuing. Without action, new triggers block on{" "} + . + + ) : ( + "Your organization has exceeded its billing limit. New triggers are blocked until you resolve this." + )} + + +
+ +
+ + Current usage: {formatCurrency(currentSpendCents / 100, false)} + + {billingLimit.effectiveAmountCents !== null && ( + + Current limit: {formatCurrency(billingLimit.effectiveAmountCents / 100, false)} + + )} + {queuedRunsLabel && {queuedRunsLabel}} + + {isRejected && queuedRunsLabel && ( + New triggers are currently blocked. + )} +
+ +
+ + + + +
+
+ + How would you like to resolve this? + + +
+ + {action === "increase" && ( + + setNewAmount(e.target.value)} + placeholder="New limit amount" + icon={ + + $ + + } + /> + + )} + {action === "increase" && fields.newAmount?.error && ( + {fields.newAmount.error} + )} +
+ + + +
+ +
+ + What should happen to queued runs? + + setResumeMode(value as typeof resumeMode)} + className="flex flex-col gap-2" + > + + + +
+ + + {isSubmitting ? "Resolving…" : "Resolve"} + + } + /> +
+
+
+ ); +} diff --git a/apps/webapp/app/components/billing/BillingLimitResolveProgress.tsx b/apps/webapp/app/components/billing/BillingLimitResolveProgress.tsx new file mode 100644 index 0000000000..50c39bf08d --- /dev/null +++ b/apps/webapp/app/components/billing/BillingLimitResolveProgress.tsx @@ -0,0 +1,26 @@ +import { AnimatedCallout } from "~/components/primitives/AnimatedCallout"; + +export function BillingLimitResolveProgress({ + show, + cancellingQueuedRuns, +}: { + show: boolean; + cancellingQueuedRuns: boolean; +}) { + if (!show) { + return null; + } + + return ( +
+ + Billing limit resolved. Environments are being unpaused — this usually takes a few seconds. + + {cancellingQueuedRuns && ( + + Cancelling queued runs across billable environments… + + )} +
+ ); +} diff --git a/apps/webapp/app/components/billing/OrgBanner.tsx b/apps/webapp/app/components/billing/OrgBanner.tsx new file mode 100644 index 0000000000..b413039390 --- /dev/null +++ b/apps/webapp/app/components/billing/OrgBanner.tsx @@ -0,0 +1,228 @@ +import { useLocation } from "@remix-run/react"; +import { DateTime } from "~/components/primitives/DateTime"; +import { environmentFullTitle } from "~/components/environments/EnvironmentLabel"; +import { AnimatedOrgBannerBar } from "~/components/billing/AnimatedOrgBannerBar"; +import { OrgBannerKind, selectOrgBanner } from "~/components/billing/selectOrgBanner"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { useEnvironment, useOptionalEnvironment } from "~/hooks/useEnvironment"; +import { + useOptionalOrganization, + useOrganization, + useBillingLimit, + useCanManageBilling, +} from "~/hooks/useOrganizations"; +import { useOptionalProject, useProject } from "~/hooks/useProject"; +import { useShowSelfServe } from "~/hooks/useShowSelfServe"; +import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; +import { v3BillingLimitsPath, v3BillingPath, v3QueuesPath } from "~/utils/pathBuilder"; +import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; + +function getUpgradeResetDate(): Date { + const nextMonth = new Date(); + nextMonth.setUTCDate(1); + nextMonth.setUTCHours(0, 0, 0, 0); + nextMonth.setUTCMonth(nextMonth.getUTCMonth() + 1); + return nextMonth; +} + +export function OrgBanner() { + const organization = useOptionalOrganization(); + const project = useOptionalProject(); + const environment = useOptionalEnvironment(); + const billingLimit = useBillingLimit(); + const currentPlan = useCurrentPlan(); + const showSelfServe = useShowSelfServe(); + const location = useLocation(); + + // Billing-limit pauses are surfaced by the dedicated limit banners (grace/rejected). + // Don't also raise the generic "environment paused" warning for them — that would + // alarm the user during the async resolve→ok window, when billing is already `ok` + // but converge hasn't finished unpausing envs yet. Manual pauses still warn. + const isPaused = !!( + organization && + project && + environment && + environment.paused && + environment.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT + ); + const isArchived = !!(organization && project && environment && environment.archivedAt); + + const bannerKind = selectOrgBanner({ + billingLimit, + hasExceededFreeTier: currentPlan?.v3Usage.hasExceededFreeTier === true, + showEnvironmentWarning: isPaused || isArchived, + showSelfServe, + }); + + const hideQueuesButton = location.pathname.endsWith("/queues"); + const hideBillingLimitBanner = location.pathname.endsWith("/settings/billing-limits"); + + switch (bannerKind) { + case OrgBannerKind.LimitRejected: + return hideBillingLimitBanner ? null : ; + case OrgBannerKind.LimitGrace: + return hideBillingLimitBanner ? null : ; + case OrgBannerKind.NoLimitConfigured: + return hideBillingLimitBanner ? null : ; + case OrgBannerKind.Upgrade: + return organization ? : null; + case OrgBannerKind.EnvironmentWarning: + return isArchived ? ( + + ) : ( + + ); + default: + return null; + } +} + +function LimitRejectedBanner() { + const organization = useOrganization(); + const showSelfServe = useShowSelfServe(); + const canManageBilling = useCanManageBilling(); + const canResolve = showSelfServe && canManageBilling; + + return ( + + Resolve + + ) : undefined + } + > + Billing limit exceeded — New triggers are currently + blocked. + {!canResolve ? " Contact your organization administrator to resolve this issue." : null} + + ); +} + +function LimitGraceBanner() { + const organization = useOrganization(); + const billingLimit = useBillingLimit(); + const showSelfServe = useShowSelfServe(); + const canManageBilling = useCanManageBilling(); + const canResolve = showSelfServe && canManageBilling; + + const graceEndsAt = + billingLimit?.isConfigured && billingLimit.limitState.status === "grace" + ? billingLimit.limitState.graceEndsAt + : null; + + return ( + + Resolve + + ) : undefined + } + > + Billing limit reached — Queues have been paused. New runs + will continue to queue until . + {!canResolve ? " Contact your organization administrator to resolve this issue." : null} + + ); +} + +function NoLimitConfiguredBanner() { + const organization = useOrganization(); + const canManageBilling = useCanManageBilling(); + + return ( + + Configure billing limit + + ) : undefined + } + > + {canManageBilling + ? "Protect your organization from unexpected usage spikes." + : "Billing limits are not configured for this organization. Contact an organization administrator to configure them."} + + ); +} + +function UpgradeBanner() { + const organization = useOrganization(); + const plan = useCurrentPlan(); + const freeCreditsDollars = (plan?.v3Subscription?.plan?.limits.includedUsage ?? 500) / 100; + + return ( + + Upgrade + + } + > + You have exceeded the monthly ${freeCreditsDollars} free credits. Existing runs will be queued + and new runs won't be created until{" "} + , or you upgrade. + + ); +} + +function PausedEnvironmentBanner({ hideButton }: { hideButton: boolean }) { + const organization = useOrganization(); + const project = useProject(); + const environment = useEnvironment(); + + return ( + + Manage + + ) + } + > + {environmentFullTitle(environment)} environment paused. No new runs will be dequeued and + executed. + + ); +} + +function ArchivedEnvironmentBanner() { + const environment = useEnvironment(); + + return ( + + "{environment.branchName}" branch is archived and is read-only. No new runs will be dequeued + and executed. + + ); +} diff --git a/apps/webapp/app/components/billing/UpgradePrompt.tsx b/apps/webapp/app/components/billing/UpgradePrompt.tsx deleted file mode 100644 index 8a3e098ba4..0000000000 --- a/apps/webapp/app/components/billing/UpgradePrompt.tsx +++ /dev/null @@ -1,53 +0,0 @@ -import { ExclamationCircleIcon } from "@heroicons/react/20/solid"; -import tileBgPath from "~/assets/images/error-banner-tile@2x.png"; -import { MatchedOrganization, useOrganization } from "~/hooks/useOrganizations"; -import { useCurrentPlan } from "~/routes/_app.orgs.$organizationSlug/route"; -import { v3BillingPath } from "~/utils/pathBuilder"; -import { LinkButton } from "../primitives/Buttons"; -import { Icon } from "../primitives/Icon"; -import { Paragraph } from "../primitives/Paragraph"; -import { DateTime } from "~/components/primitives/DateTime"; - -export function UpgradePrompt() { - const organization = useOrganization(); - const plan = useCurrentPlan(); - - if (!plan || !plan.v3Usage.hasExceededFreeTier) { - return null; - } - - const nextMonth = new Date(); - nextMonth.setUTCMonth(nextMonth.getMonth() + 1); - nextMonth.setUTCDate(1); - nextMonth.setUTCHours(0, 0, 0, 0); - - return ( -
-
- - - You have exceeded the monthly $ - {(plan.v3Subscription?.plan?.limits.includedUsage ?? 500) / 100} free credits. Existing - runs will be queued and new runs won't be created until{" "} - , or you upgrade. - -
- - Upgrade - -
- ); -} - -export function useShowUpgradePrompt(organization?: MatchedOrganization) { - const currentPlan = useCurrentPlan(); - const shouldShow = currentPlan?.v3Usage.hasExceededFreeTier === true; - return { shouldShow }; -} diff --git a/apps/webapp/app/components/billing/billingAlertsFormat.ts b/apps/webapp/app/components/billing/billingAlertsFormat.ts new file mode 100644 index 0000000000..7123681bda --- /dev/null +++ b/apps/webapp/app/components/billing/billingAlertsFormat.ts @@ -0,0 +1,379 @@ +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; + +export type BillingAlertsFormData = { + /** Stored base amount in dollars (from API cents / 100). Used when converting legacy alerts. */ + amount: number; + emails: string[]; + alertLevels: number[]; +}; + +/** $1 base (in cents) for absolute spend alerts when billing limit mode is none. */ +export const ABSOLUTE_ALERT_BASE_CENTS = 100; + +export const MAX_PERCENTAGE_ALERTS = 5; +export const MAX_ABSOLUTE_ALERTS = 10; +export const MAX_PERCENTAGE_THRESHOLD = 100; + +export type BillingLimitMode = "plan" | "custom" | "none"; + +export function getBillingLimitMode(billingLimit: BillingLimitResult): BillingLimitMode { + if (!billingLimit.isConfigured) { + return "none"; + } + return billingLimit.mode; +} + +export function isPercentageAlertMode(mode: BillingLimitMode): boolean { + return mode === "plan" || mode === "custom"; +} + +export function shouldClearAlertsOnLimitChange( + previousMode: BillingLimitMode, + nextMode: BillingLimitMode, +): boolean { + return shouldResetAlertsOnLimitChange(previousMode, nextMode); +} + +/** Alert format changes when crossing between percentage (plan/custom) and dollar (none) modes. */ +export function shouldResetAlertsOnLimitChange( + previousMode: BillingLimitMode | null, + nextMode: BillingLimitMode, +): boolean { + if (previousMode === null) { + return false; + } + + return isPercentageAlertMode(previousMode) !== isPercentageAlertMode(nextMode); +} + +/** Configured billing limit mode before a save; null when billing limit was never configured. */ +export function getPreviousBillingLimitModeForAlertSync( + billingLimit: BillingLimitResult, +): BillingLimitMode | null { + if (!billingLimit.isConfigured) { + return null; + } + + return billingLimit.mode; +} + +export function hasConfiguredAlerts( + alerts: BillingAlertsFormData, + billingLimit: BillingLimitResult, + planLimitCents: number, +): boolean { + const mode = getBillingLimitMode(billingLimit); + const effectiveLimitCents = getEffectiveLimitCents(billingLimit, planLimitCents); + return storedAlertsToThresholds(alerts, mode, effectiveLimitCents, planLimitCents).length > 0; +} + +export function hasSavedAlertThresholds(alerts: BillingAlertsFormData): boolean { + return alerts.alertLevels.length > 0; +} + +/** Saved thresholds that would be cleared when the billing limit alert format changes. */ +export function hadSavedAlertsToClearOnLimitChange( + alerts: BillingAlertsFormData, + billingLimit: BillingLimitResult, + planLimitCents: number, +): boolean { + return hasConfiguredAlerts(alerts, billingLimit, planLimitCents); +} + +export function normalizeThresholdValues(values: number[]): number[] { + return [...values].sort((a, b) => a - b); +} + +export function thresholdValuesAreUnique(values: number[]): boolean { + const normalized = normalizeThresholdValues(values); + return new Set(normalized).size === normalized.length; +} + +export function normalizeEmailValues(values: string[]): string[] { + return values.map((value) => value.trim()).filter(Boolean); +} + +export function thresholdsMatchSaved(current: number[], saved: number[]): boolean { + return ( + JSON.stringify(normalizeThresholdValues(current)) === + JSON.stringify(normalizeThresholdValues(saved)) + ); +} + +export function emailsMatchSaved(current: string[], saved: string[]): boolean { + return ( + JSON.stringify(normalizeEmailValues(current)) === JSON.stringify(normalizeEmailValues(saved)) + ); +} + +export function clearedAlertsPayload(emails: string[] = []): { + amount: number; + alertLevels: number[]; + emails: string[]; +} { + return { + amount: ABSOLUTE_ALERT_BASE_CENTS, + alertLevels: [], + emails, + }; +} + +export function resetAlertsPayloadForLimitMode( + nextMode: BillingLimitMode, + effectiveLimitCents: number, + emails: string[] = [], +): { amount: number; alertLevels: number[]; emails: string[] } { + if (nextMode === "none") { + return clearedAlertsPayload(emails); + } + + return { + amount: effectiveLimitCents, + alertLevels: [], + emails, + }; +} + +export function getEffectiveLimitCents( + billingLimit: BillingLimitResult, + planLimitCents: number, +): number { + if (!billingLimit.isConfigured) { + return planLimitCents; + } + if (billingLimit.mode === "custom") { + return billingLimit.amountCents; + } + if (billingLimit.mode === "plan") { + return billingLimit.effectiveAmountCents ?? planLimitCents; + } + return planLimitCents; +} + +/** Billing limit in cents when configured (plan/custom); undefined for none or unconfigured. */ +export function getConfiguredBillingLimitCents( + billingLimit: BillingLimitResult | undefined, + planLimitCents: number, +): number | undefined { + if (!billingLimit?.isConfigured || billingLimit.mode === "none") { + return undefined; + } + return getEffectiveLimitCents(billingLimit, planLimitCents); +} + +/** Dollars for UsageBar marker; omitted when no limit or same as plan included usage. */ +export function getUsageBarBillingLimitDollars( + billingLimit: BillingLimitResult | undefined, + planLimitCents: number, +): number | undefined { + const limitCents = getConfiguredBillingLimitCents(billingLimit, planLimitCents); + if (limitCents === undefined || limitCents === planLimitCents) { + return undefined; + } + return limitCents / 100; +} + +function getSavedAlertAmountCents(alerts: BillingAlertsFormData): number { + return Math.round(alerts.amount * 100); +} + +function usesFractionAlertLevelFormat(levels: number[]): boolean { + return levels.some((level) => level > 0 && level <= 1); +} + +function isAbsoluteDollarAlertLevels(levels: number[]): boolean { + if (levels.length === 0) { + return false; + } + + return !usesFractionAlertLevelFormat(levels); +} + +export function isAbsoluteSavedAlerts(alerts: BillingAlertsFormData): boolean { + return getSavedAlertAmountCents(alerts) === ABSOLUTE_ALERT_BASE_CENTS; +} + +/** Build a cleaned alerts payload when saving billing limits in the same alert format. */ +export function buildCleanedAlertsPayloadForLimitSave( + alerts: BillingAlertsFormData, + nextMode: BillingLimitMode, + effectiveLimitCents: number, + planLimitCents: number, +): { amount: number; alertLevels: number[]; emails: string[] } | null { + if (alerts.alertLevels.length === 0) { + return null; + } + + const thresholds = storedAlertsToThresholds( + alerts, + nextMode, + effectiveLimitCents, + planLimitCents, + ); + + return { + emails: alerts.emails, + ...thresholdsToAlertPayload(thresholds, nextMode, effectiveLimitCents), + }; +} + +/** Convert stored percentage alert levels to UI percent values (10, 50, 80). */ +export function percentageAlertLevelsToUiThresholds(levels: number[]): number[] { + const normalized = levels.filter((level) => Number.isFinite(level) && level > 0); + if (normalized.length === 0) { + return []; + } + + if (usesFractionAlertLevelFormat(normalized)) { + return normalized.filter((level) => level <= 1).map((level) => Math.round(level * 100)); + } + + return normalized + .filter((level) => level <= MAX_PERCENTAGE_THRESHOLD) + .map((level) => Math.round(level)); +} + +export function normalizeBillingAlertsFromApi(apiAlerts: { + amount: number; + emails?: string[]; + alertLevels?: number[]; +}): BillingAlertsFormData { + const rawAmount = Number(apiAlerts.amount); + const alertLevels = (apiAlerts.alertLevels ?? []).map(Number).filter(Number.isFinite); + + // Platform API stores amount in cents. + let amountDollars = rawAmount / 100; + + // Legacy percentage alerts sometimes stored plan dollars directly (e.g. 100 for $100) + // with whole-number percents (10, 50, 80). New saves store cents and fractional levels + // (0.75, 0.9) via thresholdsToAlertPayload — never treat those as legacy dollars. + if ( + rawAmount !== ABSOLUTE_ALERT_BASE_CENTS && + Number.isFinite(rawAmount) && + rawAmount >= 10 && + rawAmount / 100 < 10 && + alertLevels.length > 0 && + !usesFractionAlertLevelFormat(alertLevels) + ) { + amountDollars = rawAmount; + } + + return { + amount: amountDollars, + emails: apiAlerts.emails ?? [], + alertLevels, + }; +} + +/** Legacy alerts used plan included usage; new alerts use the billing limit amount. */ +function percentageAlertAmountMatches( + amountCents: number, + effectiveLimitCents: number, + planLimitCents: number, +): boolean { + return amountCents === effectiveLimitCents || amountCents === planLimitCents; +} + +/** Cents base for dollar preview when displaying saved percentage alerts. */ +export function getAlertPreviewLimitCents( + alerts: BillingAlertsFormData, + effectiveLimitCents: number, + planLimitCents: number, +): number { + const amountCents = getSavedAlertAmountCents(alerts); + if (amountCents > 0 && percentageAlertLevelsToUiThresholds(alerts.alertLevels).length > 0) { + return amountCents; + } + if (percentageAlertAmountMatches(amountCents, effectiveLimitCents, planLimitCents)) { + return amountCents; + } + return effectiveLimitCents; +} + +/** Convert stored API alerts to UI threshold values (percent or dollars). */ +export function storedAlertsToThresholds( + alerts: BillingAlertsFormData, + mode: BillingLimitMode, + effectiveLimitCents: number, + planLimitCents: number, +): number[] { + const amountCents = getSavedAlertAmountCents(alerts); + + if (mode === "none") { + if (alerts.alertLevels.length === 0) { + return []; + } + + // Absolute dollar alerts: API amount is the $1 base marker (100 cents). + if ( + amountCents === ABSOLUTE_ALERT_BASE_CENTS || + alerts.amount === ABSOLUTE_ALERT_BASE_CENTS / 100 + ) { + return alerts.alertLevels.slice(0, MAX_ABSOLUTE_ALERTS); + } + + return []; + } + + const uiThresholds = percentageAlertLevelsToUiThresholds(alerts.alertLevels); + if (uiThresholds.length === 0) { + return []; + } + + // Saved percentage alerts keep their thresholds whenever a positive base amount is stored. + if (amountCents > 0) { + return uiThresholds.slice(0, MAX_PERCENTAGE_ALERTS); + } + + return []; +} + +export function thresholdsToAlertPayload( + thresholds: number[], + mode: BillingLimitMode, + effectiveLimitCents: number, +): { amount: number; alertLevels: number[] } { + if (mode === "none") { + return { + amount: ABSOLUTE_ALERT_BASE_CENTS, + alertLevels: thresholds, + }; + } + + return { + amount: effectiveLimitCents, + alertLevels: thresholds.map((percent) => percent / 100), + }; +} + +export function isEmptyThreshold(value: number): boolean { + return !Number.isFinite(value) || value <= 0; +} + +export function previewDollarAmountForPercent( + percent: number, + effectiveLimitCents: number, +): number { + if (!Number.isFinite(percent) || percent <= 0) { + return 0; + } + return (effectiveLimitCents * percent) / 100 / 100; +} + +/** Legacy percentage alerts may include spike multipliers above 100%. */ +export function hasLegacySpikeAlertLevels( + alerts: BillingAlertsFormData, + mode: BillingLimitMode, + effectiveLimitCents: number, + planLimitCents: number, +): boolean { + if (!isPercentageAlertMode(mode)) { + return false; + } + + if (usesFractionAlertLevelFormat(alerts.alertLevels)) { + return alerts.alertLevels.some((level) => level > 1); + } + + return alerts.alertLevels.some((level) => level > MAX_PERCENTAGE_THRESHOLD); +} diff --git a/apps/webapp/app/components/billing/billingLimitFormat.ts b/apps/webapp/app/components/billing/billingLimitFormat.ts new file mode 100644 index 0000000000..504035392c --- /dev/null +++ b/apps/webapp/app/components/billing/billingLimitFormat.ts @@ -0,0 +1,23 @@ +import { formatDurationMilliseconds } from "@trigger.dev/core/v3"; + +/** Format billing grace period from API `gracePeriodMs` (e.g. 24 hours, not 1 day). */ +export function formatGracePeriodMs(ms: number): string { + return formatDurationMilliseconds(ms, { + style: "long", + units: ["h", "m", "s"], + maxUnits: 1, + }); +} + +export function getSuggestedRecoveryLimitDollars( + effectiveAmountCents: number | null, + currentSpendCents: number, +): number { + const candidates = [Math.ceil(currentSpendCents * 1.25)]; + if (effectiveAmountCents != null) { + candidates.push(effectiveAmountCents + 5_000, Math.ceil(effectiveAmountCents * 1.25)); + } + + const rawAmount = Math.max(...candidates) / 100; + return Math.ceil(rawAmount / 50) * 50; +} diff --git a/apps/webapp/app/components/billing/selectOrgBanner.ts b/apps/webapp/app/components/billing/selectOrgBanner.ts new file mode 100644 index 0000000000..c31c6de779 --- /dev/null +++ b/apps/webapp/app/components/billing/selectOrgBanner.ts @@ -0,0 +1,44 @@ +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; + +export enum OrgBannerKind { + LimitRejected = "limit-rejected", + LimitGrace = "limit-grace", + NoLimitConfigured = "no-limit", + Upgrade = "upgrade", + EnvironmentWarning = "env-warning", + None = "none", +} + +export function selectOrgBanner(input: { + billingLimit?: BillingLimitResult; + hasExceededFreeTier?: boolean; + showEnvironmentWarning?: boolean; + /** Self-serve billing UI — hide configure-limit prompt for managed customers. */ + showSelfServe?: boolean; +}): OrgBannerKind { + const { billingLimit, hasExceededFreeTier, showEnvironmentWarning, showSelfServe = true } = input; + + if (billingLimit?.isConfigured) { + const status = billingLimit.limitState.status; + if (status === "rejected") { + return OrgBannerKind.LimitRejected; + } + if (status === "grace") { + return OrgBannerKind.LimitGrace; + } + } + + if (hasExceededFreeTier) { + return OrgBannerKind.Upgrade; + } + + if (billingLimit && !billingLimit.isConfigured && showSelfServe) { + return OrgBannerKind.NoLimitConfigured; + } + + if (showEnvironmentWarning) { + return OrgBannerKind.EnvironmentWarning; + } + + return OrgBannerKind.None; +} diff --git a/apps/webapp/app/components/layout/AppLayout.tsx b/apps/webapp/app/components/layout/AppLayout.tsx index 3da2bfbb5a..fc7bfd7285 100644 --- a/apps/webapp/app/components/layout/AppLayout.tsx +++ b/apps/webapp/app/components/layout/AppLayout.tsx @@ -1,3 +1,4 @@ +import { forwardRef } from "react"; import { cn } from "~/utils/cn"; /** This container is used to surround the entire app, it correctly places the nav bar */ @@ -34,28 +35,28 @@ export function PageContainer({ ); } -export function PageBody({ - children, - scrollable = true, - className, -}: { - children: React.ReactNode; - scrollable?: boolean; - className?: string; -}) { +export const PageBody = forwardRef< + HTMLDivElement, + { + children: React.ReactNode; + scrollable?: boolean; + className?: string; + } +>(function PageBody({ children, scrollable = true, className }, ref) { return (
{children}
); -} +}); export function MainCenteredContainer({ children, @@ -70,14 +71,14 @@ export function MainCenteredContainer({
{children} @@ -98,7 +99,7 @@ export function MainHorizontallyCenteredContainer({
{children} diff --git a/apps/webapp/app/components/navigation/EnvironmentBanner.tsx b/apps/webapp/app/components/navigation/EnvironmentBanner.tsx deleted file mode 100644 index 2a34b9e434..0000000000 --- a/apps/webapp/app/components/navigation/EnvironmentBanner.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { ExclamationCircleIcon } from "@heroicons/react/20/solid"; -import { useLocation } from "@remix-run/react"; -import { AnimatePresence, motion } from "framer-motion"; -import { useEnvironment, useOptionalEnvironment } from "~/hooks/useEnvironment"; -import { useOptionalOrganization, useOrganization } from "~/hooks/useOrganizations"; -import { useOptionalProject, useProject } from "~/hooks/useProject"; -import { v3QueuesPath } from "~/utils/pathBuilder"; -import { environmentFullTitle } from "../environments/EnvironmentLabel"; -import { LinkButton } from "../primitives/Buttons"; -import { Icon } from "../primitives/Icon"; -import { Paragraph } from "../primitives/Paragraph"; - -export function EnvironmentBanner() { - const organization = useOptionalOrganization(); - const project = useOptionalProject(); - const environment = useOptionalEnvironment(); - - const isPaused = organization && project && environment && environment.paused; - const isArchived = organization && project && environment && environment.archivedAt; - - return ( - - {isArchived ? : isPaused ? : null} - - ); -} - -function PausedBanner() { - const organization = useOrganization(); - const project = useProject(); - const environment = useEnvironment(); - - const location = useLocation(); - const hideButton = location.pathname.endsWith("/queues"); - - return ( - -
- - - {environmentFullTitle(environment)} environment paused. No new runs will be dequeued and - executed. - -
- {hideButton ? null : ( -
- - Manage - -
- )} - - ); -} - -function ArchivedBranchBanner() { - const environment = useEnvironment(); - - return ( - -
- - - "{environment.branchName}" branch is archived and is read-only. No new runs will be - dequeued and executed. - -
- - ); -} diff --git a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx index 50b0a6d9c9..c2df01653a 100644 --- a/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx +++ b/apps/webapp/app/components/navigation/OrganizationSettingsSideMenu.tsx @@ -20,7 +20,7 @@ import { organizationTeamPath, organizationVercelIntegrationPath, rootPath, - v3BillingAlertsPath, + v3BillingLimitsPath, v3BillingPath, v3PrivateConnectionsPath, v3UsagePath, @@ -65,7 +65,7 @@ export function OrganizationSettingsSideMenu({ return (
@@ -109,12 +109,12 @@ export function OrganizationSettingsSideMenu({ /> {showSelfServe ? ( ) : null} diff --git a/apps/webapp/app/components/primitives/AnimatedCallout.tsx b/apps/webapp/app/components/primitives/AnimatedCallout.tsx new file mode 100644 index 0000000000..69bbc59f8b --- /dev/null +++ b/apps/webapp/app/components/primitives/AnimatedCallout.tsx @@ -0,0 +1,94 @@ +import { useEffect, useRef, useState } from "react"; +import { Callout, type CalloutVariant } from "~/components/primitives/Callout"; +import { cn } from "~/utils/cn"; + +const CALLOUT_ANIMATION_MS = 300; + +type AnimatedCalloutProps = { + show: boolean; + variant: CalloutVariant; + className?: string; + children: React.ReactNode; + /** When set, the callout auto-hides after this many milliseconds. */ + autoHideMs?: number; + onAutoHide?: () => void; + onHidden?: () => void; +}; + +export function AnimatedCallout({ + show, + variant, + className, + children, + autoHideMs, + onAutoHide, + onHidden, +}: AnimatedCalloutProps) { + const [rendered, setRendered] = useState(show); + const [autoDismissed, setAutoDismissed] = useState(false); + const onAutoHideRef = useRef(onAutoHide); + const onHiddenRef = useRef(onHidden); + + useEffect(() => { + onAutoHideRef.current = onAutoHide; + }, [onAutoHide]); + + useEffect(() => { + onHiddenRef.current = onHidden; + }, [onHidden]); + + const shouldShow = show && !autoDismissed; + + useEffect(() => { + if (!show) { + setAutoDismissed(false); + } + }, [show]); + + useEffect(() => { + if (shouldShow) { + setRendered(true); + return; + } + + if (!rendered) { + return; + } + + const hideTimer = window.setTimeout(() => { + setRendered(false); + onHiddenRef.current?.(); + }, CALLOUT_ANIMATION_MS); + + return () => window.clearTimeout(hideTimer); + }, [shouldShow, rendered]); + + useEffect(() => { + if (!shouldShow || autoHideMs === undefined) { + return; + } + + const closeTimer = window.setTimeout(() => { + setAutoDismissed(true); + onAutoHideRef.current?.(); + }, autoHideMs); + return () => window.clearTimeout(closeTimer); + }, [shouldShow, autoHideMs]); + + if (!rendered) { + return null; + } + + return ( +
+ {children} +
+ ); +} diff --git a/apps/webapp/app/components/primitives/PageHeader.tsx b/apps/webapp/app/components/primitives/PageHeader.tsx index 31fa8104dd..08cc068cfa 100644 --- a/apps/webapp/app/components/primitives/PageHeader.tsx +++ b/apps/webapp/app/components/primitives/PageHeader.tsx @@ -1,13 +1,11 @@ import { Link, useNavigation } from "@remix-run/react"; import { type ReactNode } from "react"; import { QuestionMarkIcon } from "~/assets/icons/QuestionMarkIcon"; -import { useOptionalOrganization } from "~/hooks/useOrganizations"; -import { UpgradePrompt, useShowUpgradePrompt } from "../billing/UpgradePrompt"; +import { OrgBanner } from "../billing/OrgBanner"; import { BreadcrumbIcon } from "./BreadcrumbIcon"; import { Header2 } from "./Headers"; import { LoadingBarDivider } from "./LoadingBarDivider"; import { SimpleTooltip } from "./Tooltip"; -import { EnvironmentBanner } from "../navigation/EnvironmentBanner"; import { DashboardAgentLauncher } from "../dashboard-agent/dashboardAgentLauncher"; type WithChildren = { @@ -16,9 +14,6 @@ type WithChildren = { }; export function NavBar({ children }: WithChildren) { - const organization = useOptionalOrganization(); - const showUpgradePrompt = useShowUpgradePrompt(organization); - const navigation = useNavigation(); const isLoading = navigation.state === "loading" || navigation.state === "submitting"; @@ -31,7 +26,7 @@ export function NavBar({ children }: WithChildren) {
- {showUpgradePrompt.shouldShow && organization ? : } +
); } diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 72191c1d0c..ab9a9b7f10 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -10,6 +10,7 @@ import { PassThrough } from "stream"; import * as Worker from "~/services/worker.server"; import { initMollifierDrainerWorker } from "~/v3/mollifierDrainerWorker.server"; import { initMollifierStaleSweepWorker } from "~/v3/mollifierStaleSweepWorker.server"; +import { initBillingLimitWorker } from "~/v3/billingLimitWorker.server"; import { bootstrap } from "./bootstrap"; import { LocaleContextProvider } from "./components/primitives/LocaleProvider"; import { @@ -41,8 +42,7 @@ import { registerRunChangeNotifierHandlers } from "./services/realtime/runChange // to globalThis is an unambiguous side effect the bundler must preserve. See // TRI-9864 for the incident write-up. import { sessionsReplicationInstance } from "./services/sessionsReplicationInstance.server"; -(globalThis as Record).__sessionsReplicationInstance = - sessionsReplicationInstance; +(globalThis as Record).__sessionsReplicationInstance = sessionsReplicationInstance; import { globalFlagsRegistry } from "./v3/globalFlagsRegistry.server"; (globalThis as Record).__globalFlagsRegistry = globalFlagsRegistry; import { workerRegionRegistry } from "./v3/workerRegions.server"; @@ -54,7 +54,7 @@ export default function handleRequest( request: Request, responseStatusCode: number, responseHeaders: Headers, - remixContext: EntryContext + remixContext: EntryContext, ) { const url = new URL(request.url); @@ -83,7 +83,7 @@ export default function handleRequest( responseHeaders, remixContext, locales, - platform + platform, ); } @@ -93,7 +93,7 @@ export default function handleRequest( responseHeaders, remixContext, locales, - platform + platform, ); } @@ -103,7 +103,7 @@ function handleBotRequest( responseHeaders: Headers, remixContext: EntryContext, locales: string[], - platform: OperatingSystemPlatform + platform: OperatingSystemPlatform, ) { return new Promise((resolve, reject) => { let shellRendered = false; @@ -129,7 +129,7 @@ function handleBotRequest( new Response(stream, { headers: responseHeaders, status: responseStatusCode, - }) + }), ); pipe(body); @@ -148,7 +148,7 @@ function handleBotRequest( console.error(error); } }, - } + }, ); abortTimer = setTimeout(abort, ABORT_DELAY); @@ -161,7 +161,7 @@ function handleBrowserRequest( responseHeaders: Headers, remixContext: EntryContext, locales: string[], - platform: OperatingSystemPlatform + platform: OperatingSystemPlatform, ) { return new Promise((resolve, reject) => { let shellRendered = false; @@ -187,7 +187,7 @@ function handleBrowserRequest( new Response(stream, { headers: responseHeaders, status: responseStatusCode, - }) + }), ); pipe(body); @@ -206,7 +206,7 @@ function handleBrowserRequest( console.error(error); } }, - } + }, ); abortTimer = setTimeout(abort, ABORT_DELAY); @@ -235,6 +235,7 @@ Worker.init().catch((error) => { initMollifierDrainerWorker(); initMollifierStaleSweepWorker(); +initBillingLimitWorker(); bootstrap().catch((error) => { logError(error); diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index e3f6ed49f5..9491fffb45 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -60,7 +60,7 @@ const GithubAppEnvSchema = z.preprocess( z.object({ GITHUB_APP_ENABLED: z.literal("0"), }), - ]) + ]), ); // eventually we can make all S2 env vars required once the S2 OSS version is out @@ -82,7 +82,7 @@ const S2EnvSchema = z.preprocess( z.object({ S2_ENABLED: z.literal("0"), }), - ]) + ]), ); const EnvironmentSchema = z @@ -92,7 +92,7 @@ const EnvironmentSchema = z .string() .refine( isValidDatabaseUrl, - "DATABASE_URL is invalid, for details please check the additional output above this message." + "DATABASE_URL is invalid, for details please check the additional output above this message.", ), DATABASE_CONNECTION_LIMIT: z.coerce.number().int().default(10), DATABASE_POOL_TIMEOUT: z.coerce.number().int().default(60), @@ -122,7 +122,7 @@ const EnvironmentSchema = z .string() .refine( isValidDatabaseUrl, - "DIRECT_URL is invalid, for details please check the additional output above this message." + "DIRECT_URL is invalid, for details please check the additional output above this message.", ), DATABASE_READ_REPLICA_URL: z.string().optional(), SESSION_SECRET: z.string(), @@ -131,7 +131,7 @@ const EnvironmentSchema = z .string() .refine( (val) => Buffer.from(val, "utf8").length === 32, - "ENCRYPTION_KEY must be exactly 32 bytes" + "ENCRYPTION_KEY must be exactly 32 bytes", ), WHITELISTED_EMAILS: z .string() @@ -204,13 +204,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RATE_LIMIT_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RATE_LIMIT_REDIS_USERNAME: z .string() @@ -236,13 +237,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), CACHE_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), CACHE_REDIS_USERNAME: z .string() @@ -263,7 +265,7 @@ const EnvironmentSchema = z .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), TASK_META_CACHE_REDIS_USERNAME: z .string() @@ -292,13 +294,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), REALTIME_STREAMS_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), REALTIME_STREAMS_REDIS_USERNAME: z .string() @@ -393,13 +396,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), PUBSUB_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), PUBSUB_REDIS_USERNAME: z .string() @@ -537,7 +541,7 @@ const EnvironmentSchema = z .string() .optional() .transform((v, ctx) => - parseMachinePresetCsv(v ?? process.env.COMPUTE_TEMPLATE_MACHINE_PRESETS ?? "small-1x", ctx) + parseMachinePresetCsv(v ?? process.env.COMPUTE_TEMPLATE_MACHINE_PRESETS ?? "small-1x", ctx), ), DEPLOY_IMAGE_PLATFORM: z.string().default("linux/amd64"), @@ -690,13 +694,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), ALERT_RATE_LIMITER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), ALERT_RATE_LIMITER_REDIS_USERNAME: z .string() @@ -917,13 +922,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RUN_ENGINE_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RUN_ENGINE_WORKER_REDIS_USERNAME: z .string() @@ -950,13 +956,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RUN_ENGINE_RUN_QUEUE_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RUN_ENGINE_RUN_QUEUE_REDIS_USERNAME: z .string() @@ -983,13 +990,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RUN_ENGINE_RUN_LOCK_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RUN_ENGINE_RUN_LOCK_REDIS_USERNAME: z .string() @@ -1016,13 +1024,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RUN_ENGINE_DEV_PRESENCE_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RUN_ENGINE_DEV_PRESENCE_REDIS_USERNAME: z .string() @@ -1120,13 +1129,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), LEGACY_RUN_ENGINE_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), LEGACY_RUN_ENGINE_WORKER_REDIS_USERNAME: z .string() @@ -1167,13 +1177,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), COMMON_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), COMMON_WORKER_REDIS_USERNAME: z .string() @@ -1222,7 +1233,7 @@ const EnvironmentSchema = z .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), TRIGGER_MOLLIFIER_REDIS_USERNAME: z .string() @@ -1378,13 +1389,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), BATCH_TRIGGER_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), BATCH_TRIGGER_WORKER_REDIS_USERNAME: z .string() @@ -1442,13 +1454,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), ADMIN_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), ADMIN_WORKER_REDIS_USERNAME: z .string() @@ -1483,13 +1496,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), ALERTS_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), ALERTS_WORKER_REDIS_USERNAME: z .string() @@ -1502,6 +1516,39 @@ const EnvironmentSchema = z ALERTS_WORKER_REDIS_TLS_DISABLED: z.string().default(process.env.REDIS_TLS_DISABLED ?? "false"), ALERTS_WORKER_REDIS_CLUSTER_MODE_ENABLED: z.string().default("0"), + BILLING_LIMIT_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), + BILLING_LIMIT_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), + BILLING_LIMIT_WORKER_CONCURRENCY_TASKS_PER_WORKER: z.coerce.number().int().default(10), + BILLING_LIMIT_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000), + BILLING_LIMIT_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50), + BILLING_LIMIT_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(20), + BILLING_LIMIT_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000), + BILLING_LIMIT_WORKER_LOG_LEVEL: z + .enum(["log", "error", "warn", "info", "debug"]) + .default("info"), + BILLING_LIMIT_RECONCILE_INTERVAL_MS: z.coerce.number().int().default(90_000), + BILLING_LIMIT_WORKER_REDIS_HOST: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_HOST), + BILLING_LIMIT_WORKER_REDIS_PORT: z.coerce + .number() + .optional() + .transform( + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), + ), + BILLING_LIMIT_WORKER_REDIS_USERNAME: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_USERNAME), + BILLING_LIMIT_WORKER_REDIS_PASSWORD: z + .string() + .optional() + .transform((v) => v ?? process.env.REDIS_PASSWORD), + BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED: z + .string() + .default(process.env.REDIS_TLS_DISABLED ?? "false"), + SCHEDULE_ENGINE_LOG_LEVEL: z.enum(["log", "error", "warn", "info", "debug"]).default("info"), SCHEDULE_WORKER_ENABLED: z.string().default(process.env.WORKER_ENABLED ?? "true"), SCHEDULE_WORKER_CONCURRENCY_WORKERS: z.coerce.number().int().default(2), @@ -1525,13 +1572,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), SCHEDULE_WORKER_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), SCHEDULE_WORKER_REDIS_USERNAME: z .string() @@ -1576,13 +1624,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), RUN_REPLICATION_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), RUN_REPLICATION_REDIS_USERNAME: z .string() @@ -1676,8 +1725,16 @@ const EnvironmentSchema = z // Bound read-in-order memory on object-storage reads: each part opens a per-column read // stream, and the default ~1 MiB+ S3 buffers dominate peak memory. These two byte sizes // cap the per-stream buffers and exist on every supported ClickHouse, so they are always on. - CLICKHOUSE_LOGS_LIST_PREFETCH_BUFFER_SIZE: z.coerce.number().int().nonnegative().default(262_144), - CLICKHOUSE_LOGS_LIST_MAX_READ_BUFFER_SIZE: z.coerce.number().int().nonnegative().default(262_144), + CLICKHOUSE_LOGS_LIST_PREFETCH_BUFFER_SIZE: z.coerce + .number() + .int() + .nonnegative() + .default(262_144), + CLICKHOUSE_LOGS_LIST_MAX_READ_BUFFER_SIZE: z.coerce + .number() + .int() + .nonnegative() + .default(262_144), // The decisive lever on Cloud SharedMergeTree, but it only exists on newer ClickHouse and // is a no-op on local-disk MergeTree, so it is opt-in: unset means it is never sent (safe on // any self-hosted version). Set to 0 on object-storage deployments to get the memory win. @@ -1699,7 +1756,7 @@ const EnvironmentSchema = z s .split(",") .map((v) => Number(v.trim())) - .filter((n) => Number.isFinite(n) && n > 0) + .filter((n) => Number.isFinite(n) && n > 0), ), // Query feature flag @@ -1769,7 +1826,10 @@ const EnvironmentSchema = z .optional() .transform((v) => v ?? process.env.CLICKHOUSE_URL), REALTIME_BACKEND_NATIVE_CLICKHOUSE_KEEP_ALIVE_ENABLED: z.string().default("1"), - REALTIME_BACKEND_NATIVE_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce.number().int().optional(), + REALTIME_BACKEND_NATIVE_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS: z.coerce + .number() + .int() + .optional(), REALTIME_BACKEND_NATIVE_CLICKHOUSE_MAX_OPEN_CONNECTIONS: z.coerce.number().int().default(10), REALTIME_BACKEND_NATIVE_CLICKHOUSE_LOG_LEVEL: z .enum(["log", "error", "warn", "info", "debug"]) @@ -1849,13 +1909,14 @@ const EnvironmentSchema = z .optional() .transform( (v) => - v ?? (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined) + v ?? + (process.env.REDIS_READER_PORT ? parseInt(process.env.REDIS_READER_PORT) : undefined), ), REQUEST_IDEMPOTENCY_REDIS_PORT: z.coerce .number() .optional() .transform( - (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined) + (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined), ), REQUEST_IDEMPOTENCY_REDIS_USERNAME: z .string() diff --git a/apps/webapp/app/hooks/useOrganizations.ts b/apps/webapp/app/hooks/useOrganizations.ts index 1aa81b1104..a546227582 100644 --- a/apps/webapp/app/hooks/useOrganizations.ts +++ b/apps/webapp/app/hooks/useOrganizations.ts @@ -87,3 +87,19 @@ export function useWidgetLimitPerDashboard(matches?: UIMatch[]) { }); return data?.widgetLimitPerDashboard ?? 16; } + +export function useBillingLimit(matches?: UIMatch[]) { + const data = useTypedMatchesData({ + id: "routes/_app.orgs.$organizationSlug", + matches, + }); + return data?.billingLimit; +} + +export function useCanManageBilling(matches?: UIMatch[]) { + const data = useTypedMatchesData({ + id: "routes/_app.orgs.$organizationSlug", + matches, + }); + return data?.canManageBilling === true; +} diff --git a/apps/webapp/app/hooks/useScrollContainerToTop.ts b/apps/webapp/app/hooks/useScrollContainerToTop.ts new file mode 100644 index 0000000000..43db10b58b --- /dev/null +++ b/apps/webapp/app/hooks/useScrollContainerToTop.ts @@ -0,0 +1,14 @@ +import { useLocation } from "@remix-run/react"; +import { useEffect, useRef } from "react"; + +/** Scroll a page body container back to the top when navigating to a route. */ +export function useScrollContainerToTop() { + const ref = useRef(null); + const location = useLocation(); + + useEffect(() => { + ref.current?.scrollTo(0, 0); + }, [location.key]); + + return ref; +} diff --git a/apps/webapp/app/models/organization.server.ts b/apps/webapp/app/models/organization.server.ts index 944d14505a..f40342f583 100644 --- a/apps/webapp/app/models/organization.server.ts +++ b/apps/webapp/app/models/organization.server.ts @@ -15,6 +15,10 @@ import { featuresForUrl } from "~/features.server"; import { createApiKeyForEnv, createPkApiKeyForEnv, envSlug } from "./api-key.server"; import { getDefaultEnvironmentConcurrencyLimit } from "~/services/platform.v3.server"; import { enqueueAttioWorkspaceSync } from "~/services/attio.server"; +import { + applyBillingLimitPauseAfterEnvCreate, + getInitialEnvPauseStateForBillingLimit, +} from "~/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server"; export type { Organization }; const nanoid = customAlphabet("1234567890abcdef", 4); @@ -53,7 +57,7 @@ export async function createOrganization( onboardingData?: Prisma.InputJsonValue; avatar?: Prisma.InputJsonValue; }, - attemptCount = 0 + attemptCount = 0, ): Promise { if (typeof process.env.BLOCKED_USERS === "string" && process.env.BLOCKED_USERS.includes(userId)) { throw new Error("Organization could not be created."); @@ -78,7 +82,7 @@ export async function createOrganization( onboardingData, avatar, }, - attemptCount + 1 + attemptCount + 1, ); } @@ -139,8 +143,9 @@ export async function createEnvironment({ const shortcode = createShortcode().join("-"); const limit = await getDefaultEnvironmentConcurrencyLimit(organization.id, type); + const billingPause = await getInitialEnvPauseStateForBillingLimit(organization.id, type); - return await prismaClient.runtimeEnvironment.create({ + const environment = await prismaClient.runtimeEnvironment.create({ data: { slug, apiKey, @@ -148,6 +153,8 @@ export async function createEnvironment({ shortcode, autoEnableInternalSources: type !== "DEVELOPMENT", maximumConcurrencyLimit: limit, + paused: billingPause.paused, + pauseSource: billingPause.pauseSource, organization: { connect: { id: organization.id, @@ -162,7 +169,15 @@ export async function createEnvironment({ type, isBranchableEnvironment, }, + include: { + organization: true, + project: true, + }, }); + + await applyBillingLimitPauseAfterEnvCreate(environment); + + return environment; } function createShortcode() { diff --git a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts index 630fa9bfb7..f674915a54 100644 --- a/apps/webapp/app/presenters/OrganizationsPresenter.server.ts +++ b/apps/webapp/app/presenters/OrganizationsPresenter.server.ts @@ -77,6 +77,7 @@ export class OrganizationsPresenter { type: true, slug: true, paused: true, + pauseSource: true, isBranchableEnvironment: true, branchName: true, parentEnvironmentId: true, @@ -101,10 +102,15 @@ export class OrganizationsPresenter { throw redirect(newProjectPath(organization)); } - const environments = fullProject. - environments.filter((env) => env.type !== "DEVELOPMENT" || env.orgMember?.userId === user.id); + const environments = fullProject.environments.filter( + (env) => env.type !== "DEVELOPMENT" || env.orgMember?.userId === user.id, + ); - const environmentsWithActivity = await hydrateEnvsWithActivity(user.id, fullProject.id, environments); + const environmentsWithActivity = await hydrateEnvsWithActivity( + user.id, + fullProject.id, + environments, + ); const environment = this.#getEnvironment({ user, @@ -206,6 +212,7 @@ export class OrganizationsPresenter { | "type" | "branchName" | "paused" + | "pauseSource" | "parentEnvironmentId" | "isBranchableEnvironment" | "archivedAt" @@ -219,7 +226,7 @@ export class OrganizationsPresenter { const env = environments.find( (e) => e.slug === environmentSlug && - (e.type !== "DEVELOPMENT" || e.orgMember?.userId === user.id) + (e.type !== "DEVELOPMENT" || e.orgMember?.userId === user.id), ); if (env) { return env; @@ -246,7 +253,7 @@ export class OrganizationsPresenter { (env) => env.type === "DEVELOPMENT" && env.parentEnvironmentId === null && - env.orgMember?.userId === user.id + env.orgMember?.userId === user.id, ); if (yourDevEnvironment) { return yourDevEnvironment; diff --git a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts index 720bfda8db..ee7b76e32d 100644 --- a/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/EnvironmentVariablesPresenter.server.ts @@ -1,12 +1,9 @@ import { $replica, PrismaClient, PrismaReplicaClient, prisma } from "~/db.server"; -import { Project } from "~/models/project.server"; -import { User } from "~/models/user.server"; +import type { Project } from "~/models/project.server"; +import type { User } from "~/models/user.server"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import type { EnvironmentVariableUpdater } from "~/v3/environmentVariables/repository"; -import { - SyncEnvVarsMapping, - EnvSlug, -} from "~/v3/vercel/vercelProjectIntegrationSchema"; +import { SyncEnvVarsMapping, EnvSlug } from "~/v3/vercel/vercelProjectIntegrationSchema"; import { VercelIntegrationService } from "~/services/vercelIntegration.server"; import { loadEnvironmentVariablesEnvironments } from "./environmentVariablesEnvironments.server"; @@ -47,7 +44,7 @@ export class EnvironmentVariablesPresenter { await loadEnvironmentVariablesEnvironments( this.#replicaClient, { userId, projectId: project.id }, - { skipProjectAccessCheck: true } + { skipProjectAccessCheck: true }, ); // Only load values for the environments we display. Projects can accumulate @@ -95,9 +92,9 @@ export class EnvironmentVariablesPresenter { "type" in lastUpdatedBy && lastUpdatedBy.type === "user" && "userId" in lastUpdatedBy && - typeof lastUpdatedBy.userId === "string" + typeof lastUpdatedBy.userId === "string", ) - .map((lastUpdatedBy) => lastUpdatedBy.userId) + .map((lastUpdatedBy) => lastUpdatedBy.userId), ); const users = @@ -117,8 +114,10 @@ export class EnvironmentVariablesPresenter { }) : []; - const usersRecord: Record = - Object.fromEntries(users.map((u) => [u.id, u])); + const usersRecord: Record< + string, + { id: string; name: string | null; displayName: string | null; avatarUrl: string | null } + > = Object.fromEntries(users.map((u) => [u.id, u])); const repository = new EnvironmentVariablesRepository(this.#prismaClient, this.#replicaClient); @@ -134,7 +133,7 @@ export class EnvironmentVariablesPresenter { const variableValuesByEnvAndKey = await repository.getVariableValuesForKeys( project.id, - nonSecretItems + nonSecretItems, ); // Get Vercel integration data if it exists @@ -146,7 +145,8 @@ export class EnvironmentVariablesPresenter { if (vercelIntegration) { vercelSyncEnvVarsMapping = vercelIntegration.parsedIntegrationData.syncEnvVarsMapping; - vercelPullEnvVarsBeforeBuild = vercelIntegration.parsedIntegrationData.config.pullEnvVarsBeforeBuild ?? null; + vercelPullEnvVarsBeforeBuild = + vercelIntegration.parsedIntegrationData.config.pullEnvVarsBeforeBuild ?? null; } return { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index b6afe46be4..c1291a9529 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -67,6 +67,7 @@ import { EnvironmentQueuePresenter } from "~/presenters/v3/EnvironmentQueuePrese import { QueueListPresenter } from "~/presenters/v3/QueueListPresenter.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; +import { ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT } from "~/utils/environmentPauseSource"; import { concurrencyPath, docsPath, @@ -149,7 +150,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( `/orgs/${params.organizationSlug}/projects/${params.projectParam}/env/${params.envParam}/queues`, request, - "Wrong method" + "Wrong method", ); } @@ -182,14 +183,22 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { } switch (action) { - case "environment-pause": + case "environment-pause": { const pauseService = new PauseEnvironmentService(); - await pauseService.call(environment, "paused"); + const result = await pauseService.call(environment, "paused"); + if (!result.success) { + return redirectWithErrorMessage(redirectPath, request, result.error); + } return redirectWithSuccessMessage(redirectPath, request, "Environment paused"); - case "environment-resume": + } + case "environment-resume": { const resumeService = new PauseEnvironmentService(); - await resumeService.call(environment, "resumed"); + const result = await resumeService.call(environment, "resumed"); + if (!result.success) { + return redirectWithErrorMessage(redirectPath, request, result.error); + } return redirectWithSuccessMessage(redirectPath, request, "Environment resumed"); + } case "queue-pause": case "queue-resume": { const friendlyId = formData.get("friendlyId"); @@ -201,21 +210,21 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const result = await queueService.call( environment, friendlyId.toString(), - action === "queue-pause" ? "paused" : "resumed" + action === "queue-pause" ? "paused" : "resumed", ); if (!result.success) { return redirectWithErrorMessage( redirectPath, request, - result.error ?? `Failed to ${action === "queue-pause" ? "pause" : "resume"} queue` + result.error ?? `Failed to ${action === "queue-pause" ? "pause" : "resume"} queue`, ); } return redirectWithSuccessMessage( redirectPath, request, - `Queue ${action === "queue-pause" ? "paused" : "resumed"}` + `Queue ${action === "queue-pause" ? "paused" : "resumed"}`, ); } case "queue-override": { @@ -235,7 +244,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { return redirectWithErrorMessage( redirectPath, request, - "Concurrency limit must be a valid number" + "Concurrency limit must be a valid number", ); } @@ -248,21 +257,21 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { environment, friendlyId.toString(), limitNumber, - user + user, ); if (!result.isOk()) { return redirectWithErrorMessage( redirectPath, request, - "Failed to override queue concurrency limit" + "Failed to override queue concurrency limit", ); } return redirectWithSuccessMessage( redirectPath, request, - "Queue concurrency limit overridden" + "Queue concurrency limit overridden", ); } case "queue-remove-override": { @@ -274,14 +283,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const result = await concurrencySystem.queues.resetConcurrencyLimit( environment, - friendlyId.toString() + friendlyId.toString(), ); if (!result.isOk()) { return redirectWithErrorMessage( redirectPath, request, - "Failed to reset queue concurrency limit" + "Failed to reset queue concurrency limit", ); } @@ -315,8 +324,8 @@ export default function Page() { environment.running === environment.concurrencyLimit * environment.burstFactor ? "limit" : environment.running > environment.concurrencyLimit - ? "burst" - : "within"; + ? "burst" + : "within"; const limitClassName = limitStatus === "burst" ? "text-warning" : limitStatus === "limit" ? "text-error" : undefined; @@ -346,7 +355,10 @@ export default function Page() { animate accessory={
- {environment.runsEnabled ? : null} + {environment.runsEnabled && + env.pauseSource !== ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT ? ( + + ) : null}
@@ -550,7 +560,7 @@ export default function Page() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - isAtQueueLimit && "text-error" + isAtQueueLimit && "text-error", )} > {queue.queued} @@ -561,7 +571,7 @@ export default function Page() { "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, queue.running > 0 && "text-text-bright", - isAtConcurrencyLimit && "text-warning" + isAtConcurrencyLimit && "text-warning", )} > {queue.running} @@ -571,7 +581,7 @@ export default function Page() { className={cn( "w-[1%] pl-16 tabular-nums", queue.paused ? "opacity-50" : undefined, - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && "font-medium text-text-bright", )} > {limit} @@ -582,7 +592,7 @@ export default function Page() { "w-[1%] pl-16", queue.paused ? "opacity-50" : undefined, isAtConcurrencyLimit && "text-warning", - queue.concurrency?.overriddenAt && "font-medium text-text-bright" + queue.concurrency?.overriddenAt && "font-medium text-text-bright", )} > {queue.concurrency?.overriddenAt ? ( @@ -676,7 +686,6 @@ export default function Page() { )} -
) : (
@@ -712,7 +721,8 @@ function EnvironmentPauseResumeButton({ }, [navigation.state]); const isLoading = Boolean( - navigation.formData?.get("action") === (env.paused ? "environment-resume" : "environment-pause") + navigation.formData?.get("action") === + (env.paused ? "environment-resume" : "environment-pause"), ); return ( @@ -749,7 +759,7 @@ function EnvironmentPauseResumeButton({ {env.paused ? `This will allow runs to be dequeued in ${environmentFullTitle(env)} again.` : `This will pause all runs from being dequeued in ${environmentFullTitle( - env + env, )}. Any executing runs will continue to run.`}
setIsOpen(false)}> @@ -893,7 +903,7 @@ function QueueOverrideConcurrencyButton({ const navigation = useNavigation(); const [isOpen, setIsOpen] = useState(false); const [concurrencyLimit, setConcurrencyLimit] = useState( - queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString() + queue.concurrencyLimit?.toString() ?? environmentConcurrencyLimit.toString(), ); const isOverridden = !!queue.concurrency?.overriddenAt; @@ -907,7 +917,7 @@ function QueueOverrideConcurrencyButton({ const isLoading = Boolean( navigation.formData?.get("action") === "queue-override" || - navigation.formData?.get("action") === "queue-remove-override" + navigation.formData?.get("action") === "queue-remove-override", ); return ( @@ -1036,7 +1046,7 @@ function EngineVersionUpgradeCallout() { export function isEnvironmentPauseResumeFormSubmission( formMethod: string | undefined, - formData: FormData | undefined + formData: FormData | undefined, ) { if (!formMethod || !formData) { return false; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx index 8943742405..a187cb5f72 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route.tsx @@ -1,396 +1,7 @@ -import { conform, list, requestIntent, useFieldList, useForm } from "@conform-to/react"; -import { parse } from "@conform-to/zod"; -import { Form, useActionData, type MetaFunction } from "@remix-run/react"; -import { json } from "@remix-run/server-runtime"; -import { tryCatch } from "@trigger.dev/core"; -import { Fragment, useEffect, useRef, useState } from "react"; -import { - type UseDataFunctionReturn, - redirect, - typedjson, - useTypedLoaderData, -} from "remix-typedjson"; -import { z } from "zod"; -import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; -import { - MainHorizontallyCenteredContainer, - PageBody, - PageContainer, -} from "~/components/layout/AppLayout"; -import { Button } from "~/components/primitives/Buttons"; -import { CheckboxWithLabel } from "~/components/primitives/Checkbox"; -import { Fieldset } from "~/components/primitives/Fieldset"; -import { FormButtons } from "~/components/primitives/FormButtons"; -import { FormError } from "~/components/primitives/FormError"; -import { Header2 } from "~/components/primitives/Headers"; -import { Input } from "~/components/primitives/Input"; -import { InputGroup } from "~/components/primitives/InputGroup"; -import { Label } from "~/components/primitives/Label"; -import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; -import { Paragraph } from "~/components/primitives/Paragraph"; -import { TextLink } from "~/components/primitives/TextLink"; -import { InfoIconTooltip } from "~/components/primitives/Tooltip"; -import { prisma } from "~/db.server"; -import { featuresForRequest } from "~/features.server"; -import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; -import { resolveOrgIdFromSlug } from "~/models/organization.server"; -import { getBillingAlerts, getCurrentPlan, setBillingAlert } from "~/services/platform.v3.server"; -import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; -import { formatCurrency, formatNumber } from "~/utils/numberFormatter"; -import { - docsPath, - OrganizationParamsSchema, - organizationPath, - v3BillingAlertsPath, - v3BillingPath, -} from "~/utils/pathBuilder"; -import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; +import { redirect, type LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { OrganizationParamsSchema, v3BillingLimitsPath } from "~/utils/pathBuilder"; -export const meta: MetaFunction = () => { - return [ - { - title: `Billing alerts | Trigger.dev`, - }, - ]; -}; - -export const loader = dashboardLoader( - { - params: OrganizationParamsSchema, - context: async (params) => { - const organizationId = await resolveOrgIdFromSlug(params.organizationSlug); - return organizationId ? { organizationId } : {}; - }, - authorization: { - action: "manage", - resource: { type: "billing" }, - message: "With your current role, you can't manage billing alerts.", - }, - }, - async ({ params, request, user }) => { - const userId = user.id; - const { organizationSlug } = params; - - const { isManagedCloud } = featuresForRequest(request); - if (!isManagedCloud) { - return redirect(organizationPath({ slug: organizationSlug })); - } - - const organization = await prisma.organization.findFirst({ - where: { slug: organizationSlug, members: { some: { userId } } }, - }); - - if (!organization) { - throw new Response(null, { status: 404, statusText: "Organization not found" }); - } - - const currentPlan = await getCurrentPlan(organization.id); - if (currentPlan?.v3Subscription?.showSelfServe === false) { - return redirect(v3BillingPath({ slug: organizationSlug })); - } - - const [error, alerts] = await tryCatch(getBillingAlerts(organization.id)); - if (error) { - throw new Response(null, { status: 404, statusText: `Billing alerts error: ${error}` }); - } - - if (!alerts) { - throw new Response(null, { status: 404, statusText: "Billing alerts not found" }); - } - - return typedjson({ - alerts: { - ...alerts, - amount: alerts.amount / 100, - }, - }); - } -); - -type BillingAlertsData = UseDataFunctionReturn; - -const schema = z.object({ - amount: z - .number({ invalid_type_error: "Not a valid amount" }) - .min(0, "Amount must be greater than 0"), - emails: z.preprocess((i) => { - if (typeof i === "string") return [i]; - - if (Array.isArray(i)) { - const emails = i.filter((v) => typeof v === "string" && v !== ""); - if (emails.length === 0) { - return [""]; - } - return emails; - } - - return [""]; - }, z.string().email().array().nonempty("At least one email is required")), - alertLevels: z.preprocess((i) => { - if (typeof i === "string") return [i]; - return i; - }, z.coerce.number().array().nonempty("At least one alert level is required")), -}); - -export const action = dashboardAction( - { - params: OrganizationParamsSchema, - context: async (params) => { - const organizationId = await resolveOrgIdFromSlug(params.organizationSlug); - return organizationId ? { organizationId } : {}; - }, - authorization: { action: "manage", resource: { type: "billing" } }, - }, - async ({ request, params, user }) => { - const userId = user.id; - const { organizationSlug } = params; - - const organization = await prisma.organization.findFirst({ - where: { slug: organizationSlug, members: { some: { userId } } }, - }); - - if (!organization) { - return redirectWithErrorMessage( - v3BillingPath({ slug: organizationSlug }), - request, - "You are not authorized to update billing alerts" - ); - } - - const currentPlan = await getCurrentPlan(organization.id); - if (currentPlan?.v3Subscription?.showSelfServe === false) { - return redirect(v3BillingPath({ slug: organizationSlug })); - } - - const formData = await request.formData(); - const submission = parse(formData, { schema }); - - if (!submission.value || submission.intent !== "submit") { - return json(submission); - } - - try { - const [error, updatedAlert] = await tryCatch( - setBillingAlert(organization.id, { - ...submission.value, - amount: submission.value.amount * 100, - }) - ); - if (error) { - return redirectWithErrorMessage( - v3BillingAlertsPath({ slug: organizationSlug }), - request, - "Failed to update billing alert" - ); - } - - if (!updatedAlert) { - return redirectWithErrorMessage( - v3BillingAlertsPath({ slug: organizationSlug }), - request, - "Failed to update billing alert" - ); - } - - return redirectWithSuccessMessage( - v3BillingAlertsPath({ slug: organizationSlug }), - request, - "Billing alert updated" - ); - } catch (error: any) { - return json({ errors: { body: error.message } }, { status: 400 }); - } - } -); - -export default function Page() { - const loaderData = useTypedLoaderData(); - - return ; -} - -function BillingAlerts({ alerts }: { alerts: BillingAlertsData["alerts"] }) { - const plan = useCurrentPlan(); - const [dollarAmount, setDollarAmount] = useState(alerts.amount.toFixed(2)); - - const lastSubmission = useActionData(); - - const [form, { emails, amount, alertLevels }] = useForm({ - id: "invite-members", - // TODO: type this - lastSubmission: lastSubmission as any, - onValidate({ formData }) { - return parse(formData, { schema }); - }, - defaultValue: { - emails: [""], - }, - }); - - const fieldValues = useRef(alerts.emails); - const emailFields = useFieldList(form.ref, { ...emails, defaultValue: alerts.emails }); - - const checkboxLevels = [0.75, 0.9, 1.0, 2.0, 5.0]; - - const spikeAlertLevels = [10.0, 20.0, 50.0, 100.0]; - - useEffect(() => { - if (alerts.emails.length > 0) { - requestIntent(form.ref.current ?? undefined, list.append(emails.name)); - } - }, [emails.name, form.ref]); - const isFree = !plan?.v3Subscription?.isPaying; - - return ( - - - - - - - - - -
-
- Billing alerts - - Receive an email when your compute spend crosses different thresholds. You can also - learn how to{" "} - - reduce your compute spend - - . - -
- -
- - - {isFree ? ( - <> - - ${dollarAmount} - - - - ) : ( - { - const numberValue = Number(e.target.value); - if (numberValue < 0) { - setDollarAmount(""); - return; - } - setDollarAmount(e.target.value); - }} - step={0.01} - min={0} - placeholder="Enter an amount" - icon={ - $ - } - className="pl-px" - fullWidth - readOnly={isFree} - /> - )} - {amount.error} - - - - {checkboxLevels.map((level) => ( - - {level * 100}%{" "} - - ({formatCurrency(Number(dollarAmount) * level, false)}) - - - } - defaultChecked={alerts.alertLevels.includes(level)} - className="pr-0" - readOnly={level === 1.0} - /> - ))} - {alertLevels.error} - - -
- - -
- {spikeAlertLevels.map((level) => ( - - {formatNumber(level * 100)}%{" "} - - ({formatCurrency(Number(dollarAmount) * level, false)}) - - - } - defaultChecked={ - alerts.alertLevels.includes(level) || - !spikeAlertLevels.some((l) => alerts.alertLevels.includes(l)) - } - className="pr-0" - /> - ))} - - - - {emailFields.map((email, index) => ( - - { - fieldValues.current[index] = e.target.value; - if ( - emailFields.length === fieldValues.current.length && - fieldValues.current.every((v) => v !== "") - ) { - requestIntent(form.ref.current ?? undefined, list.append(emails.name)); - } - }} - fullWidth - /> - {email.error} - - ))} - - - Update - - } - /> -
- -
- - - - ); +export async function loader({ params }: LoaderFunctionArgs) { + const { organizationSlug } = OrganizationParamsSchema.parse(params); + return redirect(v3BillingLimitsPath({ slug: organizationSlug }), 302); } diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation.ts new file mode 100644 index 0000000000..8edc3a9117 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation.ts @@ -0,0 +1,14 @@ +/** Revalidate org layout (billingLimit banner) after billing limits settings forms submit. */ +export function isBillingLimitSettingsFormSubmission( + formMethod: string | undefined, + formData: FormData | undefined, +): boolean { + if (!formMethod || !formData || formMethod.toLowerCase() !== "post") { + return false; + } + + const intent = formData.get("intent"); + return ( + intent === "billing-limit" || intent === "billing-alerts" || intent === "billing-limit-resolve" + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRoute.server.ts b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRoute.server.ts new file mode 100644 index 0000000000..de4b7fac13 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRoute.server.ts @@ -0,0 +1,39 @@ +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import type { ResolveBillingLimitRequest } from "~/services/billingLimit.schemas"; + +export function isEnforcementActive(billingLimit: BillingLimitResult): boolean { + return ( + billingLimit.isConfigured && + (billingLimit.limitState.status === "grace" || billingLimit.limitState.status === "rejected") + ); +} + +export function getAlertsResetRequested(request: Request): boolean { + return new URL(request.url).searchParams.get("alertsReset") === "1"; +} + +export function getEffectiveLimitCentsAfterLimitSave( + mode: "plan" | "custom" | "none", + planLimitCents: number, + customAmountDollars?: number, +): number { + if (mode === "custom") { + return Math.round((customAmountDollars ?? 0) * 100); + } + + return planLimitCents; +} + +export function getResolveSubmitted(request: Request): boolean { + return new URL(request.url).searchParams.get("resolved") === "1"; +} + +export function getSubmittedResumeMode( + request: Request, +): ResolveBillingLimitRequest["resumeMode"] | null { + const value = new URL(request.url).searchParams.get("resumeMode"); + if (value === "queue" || value === "new_only") { + return value; + } + return null; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/route.tsx new file mode 100644 index 0000000000..d1e18cd670 --- /dev/null +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.billing-limits/route.tsx @@ -0,0 +1,561 @@ +import { parse } from "@conform-to/zod"; +import type { MetaFunction } from "@remix-run/react"; +import { json, redirect } from "@remix-run/server-runtime"; +import { tryCatch } from "@trigger.dev/core"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { AdminDebugTooltip } from "~/components/admin/debugTooltip"; +import { + BillingAlertsSection, + billingAlertsSchema, + type BillingAlertsFormData, +} from "~/components/billing/BillingAlertsSection"; +import { + getBillingLimitMode, + getEffectiveLimitCents, + isPercentageAlertMode, + MAX_ABSOLUTE_ALERTS, + MAX_PERCENTAGE_ALERTS, + MAX_PERCENTAGE_THRESHOLD, + normalizeBillingAlertsFromApi, + resetAlertsPayloadForLimitMode, + shouldResetAlertsOnLimitChange, + thresholdsToAlertPayload, + hadSavedAlertsToClearOnLimitChange, + thresholdValuesAreUnique, +} from "~/components/billing/billingAlertsFormat"; +import { getSuggestedRecoveryLimitDollars } from "~/components/billing/billingLimitFormat"; +import { + BillingLimitConfigSection, + billingLimitFormSchema, +} from "~/components/billing/BillingLimitConfigSection"; +import { + BillingLimitRecoveryPanel, + billingLimitRecoveryFormSchema, +} from "~/components/billing/BillingLimitRecoveryPanel"; +import { BillingLimitResolveProgress } from "~/components/billing/BillingLimitResolveProgress"; +import { PageBody, PageContainer } from "~/components/layout/AppLayout"; +import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader"; +import { prisma } from "~/db.server"; +import { featuresForRequest } from "~/features.server"; +import { useScrollContainerToTop } from "~/hooks/useScrollContainerToTop"; +import { resolveOrgIdFromSlug } from "~/models/organization.server"; +import { + commitSession, + getSession, + redirectWithErrorMessage, + redirectWithSuccessMessage, + setSuccessMessage, +} from "~/models/message.server"; +import { + getBillingAlerts, + getBillingLimit, + getCachedUsage, + getCurrentPlan, + resolveBillingLimit, + setBillingAlert, + setBillingLimit, +} from "~/services/platform.v3.server"; +import { dashboardAction, dashboardLoader } from "~/services/routeBuilders/dashboardBuilder"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { + getAlertsResetRequested, + getEffectiveLimitCentsAfterLimitSave, + getResolveSubmitted, + getSubmittedResumeMode, + isEnforcementActive, +} from "~/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRoute.server"; +import { + countBillingLimitPausedEnvironments, + getBillingLimitQueuedRunCount, +} from "~/v3/services/billingLimit/getBillingLimitQueuedRunCount.server"; +import { + OrganizationParamsSchema, + organizationPath, + v3BillingLimitsPath, + v3BillingPath, +} from "~/utils/pathBuilder"; + +const billingLimitsAuthorization = { + action: "manage" as const, + resource: { type: "billing" as const }, +}; + +export const meta: MetaFunction = () => { + return [{ title: `Billing limits | Trigger.dev` }]; +}; + +export const loader = dashboardLoader( + { + params: OrganizationParamsSchema, + context: async (params) => { + const organizationId = await resolveOrgIdFromSlug(params.organizationSlug); + return organizationId ? { organizationId } : {}; + }, + authorization: { + ...billingLimitsAuthorization, + message: "With your current role, you can't manage billing limits.", + }, + }, + async ({ params, request, user }) => { + const userId = user.id; + const { organizationSlug } = params; + + const { isManagedCloud } = featuresForRequest(request); + if (!isManagedCloud) { + return redirect(organizationPath({ slug: organizationSlug })); + } + + const organization = await prisma.organization.findFirst({ + where: { slug: organizationSlug, members: { some: { userId } } }, + }); + + if (!organization) { + throw new Response(null, { status: 404, statusText: "Organization not found" }); + } + + const currentPlan = await getCurrentPlan(organization.id); + if (currentPlan?.v3Subscription?.showSelfServe === false) { + return redirect(v3BillingPath({ slug: organizationSlug })); + } + + const [billingLimitError, billingLimit] = await tryCatch(getBillingLimit(organization.id)); + if (billingLimitError || !billingLimit) { + throw new Response(null, { + status: 404, + statusText: `Billing limit error: ${billingLimitError ?? "not found"}`, + }); + } + + const [alertsError, alerts] = await tryCatch(getBillingAlerts(organization.id)); + if (alertsError || !alerts) { + throw new Response(null, { + status: 404, + statusText: `Billing alerts error: ${alertsError ?? "not found"}`, + }); + } + + const planLimitCents = currentPlan?.v3Subscription?.plan?.limits.includedUsage ?? 500; + const alertsResetRequested = getAlertsResetRequested(request); + const resolveSubmitted = getResolveSubmitted(request); + const submittedResumeMode = getSubmittedResumeMode(request); + + const firstDayOfMonth = new Date(); + firstDayOfMonth.setUTCDate(1); + firstDayOfMonth.setUTCHours(0, 0, 0, 0); + + const firstDayOfNextMonth = new Date(); + firstDayOfNextMonth.setUTCDate(1); + firstDayOfNextMonth.setUTCHours(0, 0, 0, 0); + firstDayOfNextMonth.setUTCMonth(firstDayOfNextMonth.getUTCMonth() + 1); + + const [usage, queuedRunCount, billingLimitPauseEnvCount] = await Promise.all([ + getCachedUsage(organization.id, { from: firstDayOfMonth, to: firstDayOfNextMonth }), + isEnforcementActive(billingLimit) + ? getBillingLimitQueuedRunCount(organization.id) + : Promise.resolve(0), + countBillingLimitPausedEnvironments(organization.id), + ]); + + const currentSpendCents = usage?.cents ?? 0; + const suggestedNewLimitDollars = isEnforcementActive(billingLimit) + ? getSuggestedRecoveryLimitDollars( + billingLimit.isConfigured ? billingLimit.effectiveAmountCents : null, + currentSpendCents, + ) + : 0; + + const alertsFormData = normalizeBillingAlertsFromApi(alerts); + + return typedjson({ + billingLimit, + alerts: alertsFormData, + planLimitCents, + isRecoveryMode: isEnforcementActive(billingLimit), + alertsResetRequested, + currentSpendCents, + queuedRunCount, + billingLimitPauseEnvCount, + resolveSubmitted, + submittedResumeMode, + suggestedNewLimitDollars, + }); + }, +); + +type LoaderData = { + billingLimit: BillingLimitResult; + alerts: BillingAlertsFormData; + planLimitCents: number; + isRecoveryMode: boolean; + alertsResetRequested: boolean; + currentSpendCents: number; + queuedRunCount: number; + billingLimitPauseEnvCount: number; + resolveSubmitted: boolean; + submittedResumeMode: "queue" | "new_only" | null; + suggestedNewLimitDollars: number; +}; + +export const action = dashboardAction( + { + params: OrganizationParamsSchema, + context: async (params) => { + const organizationId = await resolveOrgIdFromSlug(params.organizationSlug); + return organizationId ? { organizationId } : {}; + }, + authorization: billingLimitsAuthorization, + }, + async ({ request, params, user }) => { + const userId = user.id; + const { organizationSlug } = params; + + const organization = await prisma.organization.findFirst({ + where: { slug: organizationSlug, members: { some: { userId } } }, + }); + + if (!organization) { + return redirectWithErrorMessage( + v3BillingPath({ slug: organizationSlug }), + request, + "You are not authorized to update billing settings", + ); + } + + const currentPlan = await getCurrentPlan(organization.id); + if (currentPlan?.v3Subscription?.showSelfServe === false) { + return redirect(v3BillingPath({ slug: organizationSlug })); + } + + const formData = await request.formData(); + const intent = formData.get("intent"); + + if (intent === "billing-alerts") { + const submission = parse(formData, { schema: billingAlertsSchema }); + if (!submission.value || submission.intent !== "submit") { + return json({ formIntent: "billing-alerts", submission }); + } + + const [billingLimitError, billingLimit] = await tryCatch(getBillingLimit(organization.id)); + if (billingLimitError || !billingLimit) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Failed to load billing limit for alerts", + ); + } + + const planLimitCents = currentPlan?.v3Subscription?.plan?.limits.includedUsage ?? 500; + const billingLimitMode = getBillingLimitMode(billingLimit); + const maxAlerts = isPercentageAlertMode(billingLimitMode) + ? MAX_PERCENTAGE_ALERTS + : MAX_ABSOLUTE_ALERTS; + + if (submission.value.alertLevels.length > maxAlerts) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + `You can add at most ${maxAlerts} alerts`, + ); + } + + if ( + submission.value.alertLevels.some( + (threshold) => !Number.isFinite(threshold) || threshold <= 0, + ) + ) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Each alert must be greater than 0", + ); + } + + if (!thresholdValuesAreUnique(submission.value.alertLevels)) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Each alert must be unique", + ); + } + + if ( + isPercentageAlertMode(billingLimitMode) && + submission.value.alertLevels.some((threshold) => threshold > MAX_PERCENTAGE_THRESHOLD) + ) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Alerts cannot exceed 100% of your billing limit", + ); + } + + const effectiveLimitCents = getEffectiveLimitCents(billingLimit, planLimitCents); + const alertPayload = thresholdsToAlertPayload( + submission.value.alertLevels, + billingLimitMode, + effectiveLimitCents, + ); + + const [error] = await tryCatch( + setBillingAlert(organization.id, { + emails: submission.value.emails, + ...alertPayload, + }), + ); + + if (error) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Failed to update billing alerts", + ); + } + + return redirectWithSuccessMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Billing alerts updated", + ); + } + + if (intent === "billing-limit") { + const submission = parse(formData, { schema: billingLimitFormSchema }); + if (!submission.value || submission.intent !== "submit") { + return json({ formIntent: "billing-limit", submission }); + } + + const [billingLimitError, billingLimit] = await tryCatch(getBillingLimit(organization.id)); + if (billingLimitError || !billingLimit) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Failed to load billing limit", + ); + } + + if (isEnforcementActive(billingLimit)) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Resolve the active billing limit before changing settings", + ); + } + + const cancelInProgressRuns = + submission.value.mode === "none" ? false : (submission.value.cancelInProgressRuns ?? false); + const previousMode = getBillingLimitMode(billingLimit); + const resettingAlerts = shouldResetAlertsOnLimitChange(previousMode, submission.value.mode); + const planLimitCents = currentPlan?.v3Subscription?.plan?.limits.includedUsage ?? 500; + + try { + if (submission.value.mode === "custom") { + await setBillingLimit(organization.id, { + mode: "custom", + amountCents: Math.round(submission.value.amount * 100), + cancelInProgressRuns, + }); + } else if (submission.value.mode === "plan") { + await setBillingLimit(organization.id, { + mode: "plan", + cancelInProgressRuns, + }); + } else { + await setBillingLimit(organization.id, { + mode: "none", + cancelInProgressRuns: false, + }); + } + } catch (error) { + const message = error instanceof Error ? error.message : "Failed to update billing limit"; + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + message, + ); + } + + if (resettingAlerts) { + const [alertsError, existingAlerts] = await tryCatch(getBillingAlerts(organization.id)); + if (alertsError || !existingAlerts) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Billing limit updated, but failed to clear billing alerts", + ); + } + + const existingAlertsFormData = normalizeBillingAlertsFromApi(existingAlerts); + + const shouldClearSavedAlerts = hadSavedAlertsToClearOnLimitChange( + existingAlertsFormData, + billingLimit, + planLimitCents, + ); + + if (shouldClearSavedAlerts) { + const effectiveLimitCents = getEffectiveLimitCentsAfterLimitSave( + submission.value.mode, + planLimitCents, + submission.value.mode === "custom" ? submission.value.amount : undefined, + ); + + const [clearAlertsError] = await tryCatch( + setBillingAlert( + organization.id, + resetAlertsPayloadForLimitMode( + submission.value.mode, + effectiveLimitCents, + existingAlerts.emails ?? [], + ), + ), + ); + + if (clearAlertsError) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Billing limit updated, but failed to clear billing alerts", + ); + } + + const session = await getSession(request.headers.get("cookie")); + setSuccessMessage(session, "Billing limit updated"); + + return redirect(`${v3BillingLimitsPath({ slug: organizationSlug })}?alertsReset=1`, { + headers: { + "Set-Cookie": await commitSession(session), + }, + }); + } + } + + const session = await getSession(request.headers.get("cookie")); + setSuccessMessage(session, "Billing limit updated"); + + return redirect(v3BillingLimitsPath({ slug: organizationSlug }), { + headers: { + "Set-Cookie": await commitSession(session), + }, + }); + } + + if (intent === "billing-limit-resolve") { + const submission = parse(formData, { schema: billingLimitRecoveryFormSchema }); + if (!submission.value || submission.intent !== "submit") { + return json({ formIntent: "billing-limit-resolve", submission }); + } + + const [billingLimitError, billingLimit] = await tryCatch(getBillingLimit(organization.id)); + if (billingLimitError || !billingLimit) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Failed to load billing limit", + ); + } + + if (!isEnforcementActive(billingLimit)) { + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + "Billing limit is not in an enforced state", + ); + } + + const resolvePayload = + submission.value.action === "increase" + ? { + action: "increase" as const, + newAmountCents: Math.round((submission.value.newAmount ?? 0) * 100), + resumeMode: submission.value.resumeMode, + } + : { + action: "remove" as const, + resumeMode: submission.value.resumeMode, + }; + + const [error] = await tryCatch(resolveBillingLimit(organization.id, resolvePayload)); + + if (error) { + const message = error instanceof Error ? error.message : "Failed to resolve billing limit"; + return redirectWithErrorMessage( + v3BillingLimitsPath({ slug: organizationSlug }), + request, + message, + ); + } + + const session = await getSession(request.headers.get("cookie")); + setSuccessMessage(session, "Billing limit resolved"); + + const resumeModeParam = submission.value.resumeMode; + return redirect( + `${v3BillingLimitsPath({ slug: organizationSlug })}?resolved=1&resumeMode=${resumeModeParam}`, + { + headers: { + "Set-Cookie": await commitSession(session), + }, + }, + ); + } + + return json({ error: "Unknown form intent" }, { status: 400 }); + }, +); + +export default function Page() { + const { + billingLimit, + alerts, + planLimitCents, + isRecoveryMode, + alertsResetRequested, + currentSpendCents, + queuedRunCount, + billingLimitPauseEnvCount, + resolveSubmitted, + submittedResumeMode, + suggestedNewLimitDollars, + } = useTypedLoaderData(); + + const showResolveProgress = resolveSubmitted && billingLimitPauseEnvCount > 0; + const pageBodyRef = useScrollContainerToTop(); + + return ( + + + + + + + + +
+ + {isRecoveryMode && billingLimit.isConfigured ? ( + + ) : ( + + )} + +
+ + + ); +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx index a430a2b502..43476fac8a 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.settings.usage/route.tsx @@ -6,6 +6,7 @@ import { Suspense, useMemo } from "react"; import { redirect, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { URL } from "url"; import { UsageBar } from "~/components/billing/UsageBar"; +import { getUsageBarBillingLimitDollars } from "~/components/billing/billingAlertsFormat"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { Card } from "~/components/primitives/charts/Card"; import type { ChartConfig } from "~/components/primitives/charts/Chart"; @@ -30,6 +31,7 @@ import { useSearchParams } from "~/hooks/useSearchParam"; import { UsagePresenter, type UsageSeriesData } from "~/presenters/v3/UsagePresenter.server"; import { requireUserId } from "~/services/session.server"; import { formatCurrency, formatCurrencyAccurate, formatNumber } from "~/utils/numberFormatter"; +import { useBillingLimit } from "~/hooks/useOrganizations"; import { OrganizationParamsSchema, organizationPath } from "~/utils/pathBuilder"; import { useCurrentPlan } from "../_app.orgs.$organizationSlug/route"; @@ -96,6 +98,11 @@ const monthDateFormatter = new Intl.DateTimeFormat("en-US", { export default function Page() { const { usage, tasks, months, isCurrentMonth } = useTypedLoaderData(); const currentPlan = useCurrentPlan(); + const billingLimit = useBillingLimit(); + const planLimitCents = currentPlan?.v3Subscription?.plan?.limits.includedUsage ?? 0; + const billingLimitDollars = isCurrentMonth + ? getUsageBarBillingLimitDollars(billingLimit, planLimitCents) + : undefined; const { value, replace } = useSearchParams(); const month = value("month") ?? months[0].toISOString(); @@ -155,11 +162,8 @@ export default function Page() {
)} @@ -307,7 +311,7 @@ function UsageChart({ data }: { data: UsageSeriesData }) { const yAxisTickFormatter = useMemo( () => (value: number) => `$${value.toFixed(decimalPlaces)}`, - [decimalPlaces] + [decimalPlaces], ); return ( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx index 1ad36854dc..ebed894411 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug/route.tsx @@ -9,11 +9,14 @@ import { useTypedMatchesData } from "~/hooks/useTypedMatchData"; import { OrganizationsPresenter } from "~/presenters/OrganizationsPresenter.server"; import { RegionsPresenter, type Region } from "~/presenters/v3/RegionsPresenter.server"; import { getImpersonationId } from "~/services/impersonation.server"; -import { getCachedUsage, getCurrentPlan } from "~/services/platform.v3.server"; +import { getCachedUsage, getBillingLimit, getCurrentPlan } from "~/services/platform.v3.server"; +import { rbac } from "~/services/rbac.server"; +import { canManageBilling } from "~/services/routeBuilders/permissions.server"; import { requireUser } from "~/services/session.server"; import { telemetry } from "~/services/telemetry.server"; import { organizationPath } from "~/utils/pathBuilder"; import { isEnvironmentPauseResumeFormSubmission } from "../_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route"; +import { isBillingLimitSettingsFormSubmission } from "../_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation"; const ParamsSchema = z.object({ organizationSlug: z.string(), @@ -53,6 +56,10 @@ export const shouldRevalidate: ShouldRevalidateFunction = (params) => { return true; } + if (isBillingLimitSettingsFormSubmission(params.formMethod, params.formData)) { + return true; + } + // This prevents revalidation when there are search params changes // IMPORTANT: If the loader function depends on search params, this should be updated return params.currentUrl.pathname !== params.nextUrl.pathname; @@ -85,16 +92,22 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // Using the 1st day of next month means we get the usage for the current month // and the cache key for getCachedUsage is stable over the month const firstDayOfNextMonth = new Date(); - firstDayOfNextMonth.setUTCMonth(firstDayOfNextMonth.getUTCMonth() + 1); firstDayOfNextMonth.setUTCDate(1); firstDayOfNextMonth.setUTCHours(0, 0, 0, 0); + firstDayOfNextMonth.setUTCMonth(firstDayOfNextMonth.getUTCMonth() + 1); - const shouldLoadRegions = - !!projectParam && !!environment && environment.type !== "DEVELOPMENT"; + const shouldLoadRegions = !!projectParam && !!environment && environment.type !== "DEVELOPMENT"; - const [plan, usage, customDashboards, regions] = await Promise.all([ + const [sessionAuth, plan, usage, billingLimit, customDashboards, regions] = await Promise.all([ + rbac + .authenticateSession(request, { + userId: user.id, + organizationId: organization.id, + }) + .catch(() => ({ ok: false as const, reason: "unauthorized" as const })), getCurrentPlan(organization.id), getCachedUsage(organization.id, { from: firstDayOfMonth, to: firstDayOfNextMonth }), + getBillingLimit(organization.id), prisma.metricsDashboard.findMany({ where: { organizationId: organization.id }, select: { @@ -111,6 +124,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { .catch(() => [] as Region[]) : Promise.resolve([] as Region[]), ]); + const userCanManageBilling = sessionAuth.ok ? canManageBilling(sessionAuth.ability) : false; let hasExceededFreeTier = false; let usagePercentage = 0; @@ -124,14 +138,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const dashboardLimit = typeof metricDashboardsLimitValue === "number" ? metricDashboardsLimitValue - : metricDashboardsLimitValue?.number ?? 3; + : (metricDashboardsLimitValue?.number ?? 3); // Derive widget-per-dashboard limit from plan, fallback to 16 const metricWidgetsLimitValue = plan?.v3Subscription?.plan?.limits?.metricWidgetsPerDashboard; const widgetLimitPerDashboard = typeof metricWidgetsLimitValue === "number" ? metricWidgetsLimitValue - : metricWidgetsLimitValue?.number ?? 16; + : (metricWidgetsLimitValue?.number ?? 16); // Compute widget counts per dashboard from layout JSON const customDashboardsWithWidgetCount = customDashboards.map((d) => { @@ -160,12 +174,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { regions, isImpersonating: !!impersonationId, currentPlan: { ...plan, v3Usage: { ...usage, hasExceededFreeTier, usagePercentage } }, + billingLimit, customDashboards: customDashboardsWithWidgetCount, dashboardLimits: { used: customDashboards.length, limit: dashboardLimit, }, widgetLimitPerDashboard, + canManageBilling: userCanManageBilling, }); }; diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts new file mode 100644 index 0000000000..ef3f8ba025 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.hit.ts @@ -0,0 +1,67 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { + BillingLimitHitWebhookBodySchema, + type BillingLimitHitWebhookBody, +} from "~/services/billingLimit.schemas"; +import { logger } from "~/services/logger.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { bustBillingLimitCaches } from "~/services/platform.v3.server"; +import { + enqueueBillingLimitCancelInProgressRuns, + enqueueBillingLimitConverge, +} from "~/v3/billingLimitWorker.server"; +import { BillingLimitConvergeEnvironmentsService } from "~/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server"; +import { processBillingLimitHit } from "~/v3/services/billingLimit/billingLimitHit.server"; + +const ParamsSchema = z.object({ + organizationId: z.string(), +}); + +/** Billing platform webhook: org entered billing limit grace. Idempotent — returns 202. */ +export async function action({ request, params }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toLowerCase() !== "post") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const { organizationId } = ParamsSchema.parse(params); + + let body: BillingLimitHitWebhookBody; + try { + body = BillingLimitHitWebhookBodySchema.parse(await request.json()); + } catch (error) { + logger.error("Invalid billing limit hit webhook payload", { + error, + organizationId, + }); + return json({ error: "Invalid request body" }, { status: 400 }); + } + + const organization = await prisma.organization.findFirst({ + where: { id: organizationId }, + select: { id: true }, + }); + + if (!organization) { + return json({ error: "Organization not found" }, { status: 404 }); + } + + await processBillingLimitHit( + { + organizationId, + hitAt: body.hitAt, + cancelInProgressRuns: body.cancelInProgressRuns, + }, + { + bustCaches: bustBillingLimitCaches, + seedReconcileQueue: BillingLimitConvergeEnvironmentsService.seedReconcileQueue, + enqueueConverge: enqueueBillingLimitConverge, + enqueueCancelInProgressRuns: enqueueBillingLimitCancelInProgressRuns, + }, + ); + + return json({ success: true, accepted: true }, { status: 202 }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts new file mode 100644 index 0000000000..de4362e5ef --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.reject.ts @@ -0,0 +1,37 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { bustBillingLimitCaches } from "~/services/platform.v3.server"; +import { BillingLimitConvergeEnvironmentsService } from "~/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server"; +import { enqueueBillingLimitConverge } from "~/v3/billingLimitWorker.server"; + +const ParamsSchema = z.object({ + organizationId: z.string(), +}); + +/** Billing platform webhook: org billing limit grace expired. Idempotent — returns 202. */ +export async function action({ request, params }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toLowerCase() !== "post") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const { organizationId } = ParamsSchema.parse(params); + + const organization = await prisma.organization.findFirst({ + where: { id: organizationId }, + select: { id: true }, + }); + + if (!organization) { + return json({ error: "Organization not found" }, { status: 404 }); + } + + bustBillingLimitCaches(organizationId); + await BillingLimitConvergeEnvironmentsService.seedReconcileQueue(organizationId); + await enqueueBillingLimitConverge(organizationId, "rejected"); + + return json({ success: true, accepted: true }, { status: 202 }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts new file mode 100644 index 0000000000..e41519ec81 --- /dev/null +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.billing-limit.resolve.ts @@ -0,0 +1,70 @@ +import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; +import { z } from "zod"; +import { prisma } from "~/db.server"; +import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; +import { bustBillingLimitCaches } from "~/services/platform.v3.server"; +import { logger } from "~/services/logger.server"; +import { enqueueBillingLimitResolve } from "~/v3/billingLimitWorker.server"; +import { processBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitResolve.server"; +import type { PendingBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitPendingResolve.types"; + +const ParamsSchema = z.object({ + organizationId: z.string(), +}); + +const BodySchema = z.object({ + resumeMode: z.enum(["queue", "new_only"]), + resolvedAt: z.string(), +}); + +/** Billing platform webhook: org resolved billing limit to ok. Idempotent — returns 202. */ +export async function action({ request, params }: ActionFunctionArgs) { + await requireAdminApiRequest(request); + + if (request.method.toLowerCase() !== "post") { + return json({ error: "Method not allowed" }, { status: 405 }); + } + + const { organizationId } = ParamsSchema.parse(params); + + const organization = await prisma.organization.findFirst({ + where: { id: organizationId }, + select: { id: true }, + }); + + if (!organization) { + return json({ error: "Organization not found" }, { status: 404 }); + } + + let pending: PendingBillingLimitResolve; + try { + const body = await request.json(); + pending = { + organizationId, + ...BodySchema.parse(body), + }; + } catch (error) { + logger.error("Invalid billing limit resolve webhook payload", { + error, + organizationId, + }); + return json({ error: "Invalid request body" }, { status: 400 }); + } + + try { + await processBillingLimitResolve(pending, { + bustCaches: bustBillingLimitCaches, + enqueueResolve: enqueueBillingLimitResolve, + }); + } catch (error) { + logger.error("Billing limit resolve webhook failed", { + error, + organizationId, + resumeMode: pending.resumeMode, + resolvedAt: pending.resolvedAt, + }); + return json({ error: "Failed to process billing limit resolve" }, { status: 500 }); + } + + return json({ success: true, accepted: true }, { status: 202 }); +} diff --git a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts index d60754f046..f5c3687364 100644 --- a/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts +++ b/apps/webapp/app/routes/admin.api.v1.orgs.$organizationId.runs.enable.ts @@ -1,5 +1,6 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { + EnvironmentPauseSource, type RuntimeEnvironment, type Organization, type Project, @@ -21,7 +22,12 @@ const BodySchema = z.object({ }); /** - * It will enabled/disable runs + * Enable or disable runs for an organization and pause/resume its non-dev environments. + * + * Billing-limit-paused environments are left unchanged when enabling or disabling runs; + * they are reported in `skipped`, not counted in the update total. Other per-environment + * failures are returned in `failures` (HTTP 409 when every environment fails, otherwise + * HTTP 200). */ export async function action({ request, params }: ActionFunctionArgs) { await requireAdminApiRequest(request); @@ -59,20 +65,58 @@ export async function action({ request, params }: ActionFunctionArgs) { }); const pauseEnvironmentService = new PauseEnvironmentService(); + const pauseAction = body.data.enable ? "resumed" : "paused"; + const failures: Array<{ environmentId: string; error: string }> = []; + const skipped: Array<{ environmentId: string; reason: string }> = []; + let updatedCount = 0; - // Set the organization.runsEnabled flag to false for (const environment of environments) { - if (body.data.enable) { - await pauseEnvironmentService.call({ ...environment, organization }, "resumed"); + if (environment.pauseSource === EnvironmentPauseSource.BILLING_LIMIT) { + if (!body.data.enable) { + skipped.push({ + environmentId: environment.id, + reason: "Environment is already paused due to billing limit and was left unchanged.", + }); + continue; + } + + skipped.push({ + environmentId: environment.id, + reason: + "Environment is paused due to billing limit and was left unchanged. Resolve the billing limit to resume.", + }); + continue; + } + + const result = await pauseEnvironmentService.call( + { ...environment, organization }, + pauseAction, + ); + if (result.success) { + updatedCount++; } else { - await pauseEnvironmentService.call({ ...environment, organization }, "paused"); + failures.push({ environmentId: environment.id, error: result.error }); } } + const stateLabel = body.data.enable ? "enabled" : "disabled"; + const message = `${updatedCount} of ${environments.length} environments updated to ${stateLabel}`; + + if (failures.length > 0) { + return json( + { + success: false, + message, + failures, + ...(skipped.length > 0 ? { skipped } : {}), + }, + { status: updatedCount === 0 ? 409 : 200 }, + ); + } + return json({ success: true, - message: `${environments.length} environments updated to ${ - body.data.enable ? "enabled" : "disabled" - }`, + message, + ...(skipped.length > 0 ? { skipped } : {}), }); } diff --git a/apps/webapp/app/routes/storybook.callout/route.tsx b/apps/webapp/app/routes/storybook.callout/route.tsx index d5e3464dae..be73173d00 100644 --- a/apps/webapp/app/routes/storybook.callout/route.tsx +++ b/apps/webapp/app/routes/storybook.callout/route.tsx @@ -1,11 +1,26 @@ import { EnvelopeIcon } from "@heroicons/react/20/solid"; +import { useState } from "react"; +import { AnimatedCallout } from "~/components/primitives/AnimatedCallout"; +import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Header2 } from "~/components/primitives/Headers"; export default function Story() { + const [showAnimatedCallout, setShowAnimatedCallout] = useState(true); + return (
+ Animated callout + + + This callout fades in and out + Callouts This is an info callout This is a warning callout diff --git a/apps/webapp/app/runEngine/validators/triggerTaskValidator.ts b/apps/webapp/app/runEngine/validators/triggerTaskValidator.ts index ab6e46521a..60ce8f586c 100644 --- a/apps/webapp/app/runEngine/validators/triggerTaskValidator.ts +++ b/apps/webapp/app/runEngine/validators/triggerTaskValidator.ts @@ -1,7 +1,7 @@ import { MAX_TAGS_PER_RUN } from "~/models/taskRunTag.server"; import { logger } from "~/services/logger.server"; import { getEntitlement } from "~/services/platform.v3.server"; -import { MAX_ATTEMPTS, OutOfEntitlementError } from "~/v3/services/triggerTask.server"; +import { MAX_ATTEMPTS } from "~/v3/services/triggerTask.server"; import { isFinalRunStatus } from "~/v3/taskStatus"; import type { EntitlementValidationParams, @@ -12,6 +12,7 @@ import type { TriggerTaskValidator, ValidationResult, } from "../types"; +import { validateProductionEntitlement } from "./validateProductionEntitlement.server"; import { ServiceValidationError } from "~/v3/services/common.server"; export class DefaultTriggerTaskValidator implements TriggerTaskValidator { @@ -30,7 +31,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator { return { ok: false, error: new ServiceValidationError( - `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${tags.length}.` + `Runs can only have ${MAX_TAGS_PER_RUN} tags, you're trying to set ${tags.length}.`, ), }; } @@ -39,24 +40,9 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator { } async validateEntitlement( - params: EntitlementValidationParams + params: EntitlementValidationParams, ): Promise { - const { environment } = params; - - if (environment.type === "DEVELOPMENT") { - return { ok: true }; - } - - const result = await getEntitlement(environment.organizationId); - - if (result && result.hasAccess === false) { - return { - ok: false, - error: new OutOfEntitlementError(), - }; - } - - return { ok: true, plan: result?.plan }; + return validateProductionEntitlement(params, getEntitlement); } validateMaxAttempts(params: MaxAttemptsValidationParams): ValidationResult { @@ -66,7 +52,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator { return { ok: false, error: new ServiceValidationError( - `Failed to trigger ${taskId} after ${MAX_ATTEMPTS} attempts.` + `Failed to trigger ${taskId} after ${MAX_ATTEMPTS} attempts.`, ), }; } @@ -96,7 +82,7 @@ export class DefaultTriggerTaskValidator implements TriggerTaskValidator { return { ok: false, error: new ServiceValidationError( - `Cannot trigger ${taskId} as the parent run has a status of ${parentRun.status}` + `Cannot trigger ${taskId} as the parent run has a status of ${parentRun.status}`, ), }; } diff --git a/apps/webapp/app/runEngine/validators/validateProductionEntitlement.server.ts b/apps/webapp/app/runEngine/validators/validateProductionEntitlement.server.ts new file mode 100644 index 0000000000..937862e997 --- /dev/null +++ b/apps/webapp/app/runEngine/validators/validateProductionEntitlement.server.ts @@ -0,0 +1,27 @@ +import type { EntitlementResult } from "~/services/billingLimit.schemas"; +import { OutOfEntitlementError } from "~/v3/outOfEntitlementError.server"; +import type { EntitlementValidationParams, EntitlementValidationResult } from "../types"; + +export type GetEntitlementFn = (organizationId: string) => Promise; + +export async function validateProductionEntitlement( + params: EntitlementValidationParams, + getEntitlementFn: GetEntitlementFn, +): Promise { + const { environment } = params; + + if (environment.type === "DEVELOPMENT") { + return { ok: true }; + } + + const result = await getEntitlementFn(environment.organizationId); + + if (result && result.hasAccess === false) { + return { + ok: false, + error: new OutOfEntitlementError(), + }; + } + + return { ok: true, plan: result?.plan }; +} diff --git a/apps/webapp/app/services/billingLimit.schemas.ts b/apps/webapp/app/services/billingLimit.schemas.ts new file mode 100644 index 0000000000..471d3c0553 --- /dev/null +++ b/apps/webapp/app/services/billingLimit.schemas.ts @@ -0,0 +1,181 @@ +import { BillingClient } from "@trigger.dev/platform"; +import { z } from "zod"; + +/** + * Billing limit API schemas for the billing platform service. + * + * These mirror the planned @trigger.dev/platform types and are used via + * BillingClient.fetch until the platform package is published with native + * BillingClient methods. + */ + +export const BillingLimitStateSchema = z.discriminatedUnion("status", [ + z.object({ + status: z.literal("ok"), + }), + z.object({ + status: z.literal("grace"), + hitAt: z.string().datetime({ offset: true }), + graceEndsAt: z.string().datetime({ offset: true }), + }), + z.object({ + status: z.literal("rejected"), + hitAt: z.string().datetime({ offset: true }), + graceEndsAt: z.string().datetime({ offset: true }), + }), +]); + +export type BillingLimitState = z.infer; + +export const BillingLimitConfigSchema = z.discriminatedUnion("mode", [ + z.object({ + mode: z.literal("none"), + }), + z.object({ + mode: z.literal("plan"), + }), + z.object({ + mode: z.literal("custom"), + amountCents: z.number().int().positive(), + }), +]); + +export type BillingLimitConfig = z.infer; + +export const BillingLimitUnconfiguredSchema = z.object({ + isConfigured: z.literal(false), + gracePeriodMs: z.number().int().nonnegative(), +}); + +const billingLimitConfiguredFields = { + isConfigured: z.literal(true), + cancelInProgressRuns: z.boolean(), + limitState: BillingLimitStateSchema, + effectiveAmountCents: z.number().int().nonnegative().nullable(), + gracePeriodMs: z.number().int().nonnegative(), +}; + +export const BillingLimitConfiguredNoneSchema = z.object({ + ...billingLimitConfiguredFields, + mode: z.literal("none"), +}); + +export const BillingLimitConfiguredPlanSchema = z.object({ + ...billingLimitConfiguredFields, + mode: z.literal("plan"), +}); + +export const BillingLimitConfiguredCustomSchema = z.object({ + ...billingLimitConfiguredFields, + mode: z.literal("custom"), + amountCents: z.number().int().positive(), +}); + +export const BillingLimitConfiguredSchema = z.discriminatedUnion("mode", [ + BillingLimitConfiguredNoneSchema, + BillingLimitConfiguredPlanSchema, + BillingLimitConfiguredCustomSchema, +]); + +export const BillingLimitResultSchema = z.union([ + BillingLimitUnconfiguredSchema, + BillingLimitConfiguredNoneSchema, + BillingLimitConfiguredPlanSchema, + BillingLimitConfiguredCustomSchema, +]); + +export type BillingLimitResult = z.infer; + +export const UpdateBillingLimitRequestSchema = z.discriminatedUnion("mode", [ + z.object({ + mode: z.literal("none"), + cancelInProgressRuns: z.boolean(), + }), + z.object({ + mode: z.literal("plan"), + cancelInProgressRuns: z.boolean(), + }), + z.object({ + mode: z.literal("custom"), + amountCents: z.number().int().positive(), + cancelInProgressRuns: z.boolean(), + }), +]); + +export type UpdateBillingLimitRequest = z.infer; + +export const ResolveBillingLimitRequestSchema = z.discriminatedUnion("action", [ + z.object({ + action: z.literal("increase"), + newAmountCents: z.number().int().positive(), + resumeMode: z.enum(["queue", "new_only"]), + }), + z.object({ + action: z.literal("remove"), + resumeMode: z.enum(["queue", "new_only"]), + }), +]); + +export type ResolveBillingLimitRequest = z.infer; + +export const BillingLimitActiveOrgSchema = z.object({ + orgId: z.string(), + limitState: z.enum(["grace", "rejected"]), +}); + +export const BillingLimitsActiveResultSchema = z.object({ + orgs: z.array(BillingLimitActiveOrgSchema), +}); + +export type BillingLimitsActiveResult = z.infer; + +export const BillingLimitPendingResolveOrgSchema = z.object({ + organizationId: z.string(), + resumeMode: z.enum(["queue", "new_only"]), + resolvedAt: z.string().datetime({ offset: true }), +}); + +export const BillingLimitsPendingResolvesResultSchema = z.object({ + orgs: z.array(BillingLimitPendingResolveOrgSchema), +}); + +export type BillingLimitsPendingResolvesResult = z.infer< + typeof BillingLimitsPendingResolvesResultSchema +>; + +export const BillingLimitHitWebhookBodySchema = z.object({ + hitAt: z.string().datetime({ offset: true }), + cancelInProgressRuns: z.boolean(), + limitState: z.literal("grace"), +}); + +export type BillingLimitHitWebhookBody = z.infer; + +/** Entitlement response — mirrors ReportUsageResult with billing limit fields until platform ships native types. */ +export const EntitlementResultSchema = z.object({ + hasAccess: z.boolean(), + balance: z.number().optional(), + usage: z.number().optional(), + overage: z.number().optional(), + plan: z + .object({ + type: z.string(), + code: z.string(), + isPaying: z.boolean(), + }) + .optional(), + limitState: z.literal("grace").optional(), + reason: z.enum(["free_tier_exceeded", "billing_limit"]).optional(), +}); + +export type EntitlementResult = z.infer; + +export type BillingLimitPageData = BillingLimitResult & { + queuedRunCount: number; + currentSpendCents: number; +}; + +/** Bridge webapp Zod schemas to BillingClient.fetch (separate Zod type instances). */ +export function asPlatformSchema(schema: z.ZodTypeAny) { + return schema as unknown as Parameters[1]; +} diff --git a/apps/webapp/app/services/platform.v3.server.ts b/apps/webapp/app/services/platform.v3.server.ts index fdc709fb1b..46daceec43 100644 --- a/apps/webapp/app/services/platform.v3.server.ts +++ b/apps/webapp/app/services/platform.v3.server.ts @@ -11,13 +11,27 @@ import { type PrivateLinkConnection, type PrivateLinkConnectionList, type PrivateLinkRegionsResult, - type ReportUsageResult, type SetPlanBody, type UpdateBillingAlertsRequest, type UsageResult, type UsageSeriesParams, type CurrentPlan, } from "@trigger.dev/platform"; +import { + BillingLimitResultSchema, + BillingLimitsActiveResultSchema, + BillingLimitsPendingResolvesResultSchema, + EntitlementResultSchema, + ResolveBillingLimitRequestSchema, + UpdateBillingLimitRequestSchema, + asPlatformSchema, + type BillingLimitResult, + type BillingLimitsActiveResult, + type BillingLimitsPendingResolvesResult, + type EntitlementResult, + type ResolveBillingLimitRequest, + type UpdateBillingLimitRequest, +} from "~/services/billingLimit.schemas"; import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache"; import { createLRUMemoryStore } from "@internal/cache"; import { existsSync, readFileSync } from "node:fs"; @@ -62,16 +76,14 @@ const platformClientMeter = metrics.getMeter("trigger.dev/platform-client"); const platformClientFailuresCounter = platformClientMeter.createCounter( "platform_client.failures_total", { - description: - "Failures returned or thrown by @trigger.dev/platform billing client calls", - } + description: "Failures returned or thrown by @trigger.dev/platform billing client calls", + }, ); function recordPlatformFailure(fn: string, kind: "caught" | "no_success") { platformClientFailuresCounter.add(1, { function: fn, kind }); } - function initializePlatformCache() { const ctx = new DefaultStatefulContext(); const memory = createLRUMemoryStore(1000); @@ -99,11 +111,16 @@ function initializePlatformCache() { fresh: 60_000 * 5, // 5 minutes stale: 60_000 * 10, // 10 minutes }), - entitlement: new Namespace(ctx, { + entitlement: new Namespace(ctx, { stores: [memory, redisCacheStore], fresh: 60_000, // serve without revalidation for 60s stale: 120_000, // total TTL — fresh 0-60s, stale-revalidate 60-120s }), + billingLimit: new Namespace(ctx, { + stores: [memory, redisCacheStore], + fresh: 60_000, + stale: 120_000, + }), }); return cache; @@ -111,6 +128,15 @@ function initializePlatformCache() { const platformCache = singleton("platformCache", initializePlatformCache); +function invalidateBillingLimitCaches(organizationId: string) { + platformCache.billingLimit.remove(organizationId).catch(() => {}); + platformCache.entitlement.remove(organizationId).catch(() => {}); +} + +export function bustBillingLimitCaches(organizationId: string) { + invalidateBillingLimitCaches(organizationId); +} + type Machines = typeof machinesFromPlatform; const MachineOverrideValues = z.object({ @@ -262,7 +288,7 @@ export type SelfServePurchaseBlockReason = "plan_unavailable" | "managed_billing * if the current plan can't be loaded or the org is on managed billing. */ export function getSelfServePurchaseBlockReason( - currentPlan: Awaited> + currentPlan: Awaited>, ): SelfServePurchaseBlockReason | undefined { if (!isBillingConfigured()) { return undefined; @@ -309,7 +335,7 @@ export async function getLimit(orgId: string, limit: keyof Limits, fallback: num export async function getDefaultEnvironmentConcurrencyLimit( organizationId: string, - environmentType: RuntimeEnvironmentType + environmentType: RuntimeEnvironmentType, ): Promise { if (!client) { const org = await $replica.organization.findFirst({ @@ -335,7 +361,7 @@ export async function getDefaultEnvironmentConcurrencyLimit( export function getDefaultEnvironmentLimitFromPlan( environmentType: RuntimeEnvironmentType, - plan: CurrentPlan + plan: CurrentPlan, ): number | undefined { if (!plan.v3Subscription?.plan) return undefined; @@ -393,7 +419,7 @@ export async function setPlan( request: Request, callerPath: string, plan: SetPlanBody, - opts?: { invalidateBillingCache?: (orgId: string) => void } + opts?: { invalidateBillingCache?: (orgId: string) => void }, ) { if (!client) { return redirectWithErrorMessage(callerPath, request, "Error setting plan", { @@ -432,7 +458,7 @@ export async function setPlan( callerPath, request, "Free tier unlock failed, your GitHub account is too new.", - { ephemeral: false } + { ephemeral: false }, ); } } @@ -536,20 +562,25 @@ export async function getUsage(organizationId: string, { from, to }: { from: Dat export async function getCachedUsage( organizationId: string, - { from, to }: { from: Date; to: Date } + { from, to }: { from: Date; to: Date }, ) { if (!client) return undefined; - const result = await platformCache.usage.swr( - `${organizationId}:${from.toISOString()}:${to.toISOString()}`, - async () => { - const usageResponse = await getUsage(organizationId, { from, to }); + try { + const result = await platformCache.usage.swr( + `${organizationId}:${from.toISOString()}:${to.toISOString()}`, + async () => { + const usageResponse = await getUsage(organizationId, { from, to }); - return usageResponse; - } - ); + return usageResponse; + }, + ); - return result.val; + return result.val; + } catch (e) { + recordPlatformFailure("getCachedUsage", "caught"); + return undefined; + } } export async function getUsageSeries(organizationId: string, params: UsageSeriesParams) { @@ -571,7 +602,7 @@ export async function getUsageSeries(organizationId: string, params: UsageSeries export async function reportInvocationUsage( organizationId: string, costInCents: number, - additionalData?: Record + additionalData?: Record, ) { if (!client) return undefined; @@ -603,8 +634,8 @@ export async function reportComputeUsage(request: Request) { } export async function getEntitlement( - organizationId: string -): Promise { + organizationId: string, +): Promise { if (!client) return undefined; // Errors must be caught inside the loader — @unkey/cache passes the loader @@ -616,7 +647,10 @@ export async function getEntitlement( // SWR call so it never becomes a cached access decision. const result = await platformCache.entitlement.swr(organizationId, async () => { try { - const response = await client.getEntitlement(organizationId); + const response = await client.fetch( + `/api/v1/orgs/${organizationId}/usage/entitlement`, + asPlatformSchema(EntitlementResultSchema), + ); if (!response.success) { recordPlatformFailure("getEntitlement", "no_success"); return undefined; @@ -637,8 +671,162 @@ export async function getEntitlement( return result.val; } +export async function getBillingLimit( + organizationId: string, +): Promise { + if (!client) return undefined; + + // Loader callback errors are caught below; also guard the SWR read itself so + // Redis/cache infra failures cannot reject org-layout Promise.all callers. + try { + const result = await platformCache.billingLimit.swr(organizationId, async () => { + try { + const response = await client.fetch( + `/api/v1/orgs/${organizationId}/billing-limit`, + asPlatformSchema(BillingLimitResultSchema), + ); + if (!response.success) { + recordPlatformFailure("getBillingLimit", "no_success"); + return undefined; + } + return response; + } catch (e) { + recordPlatformFailure("getBillingLimit", "caught"); + return undefined; + } + }); + + if (result.err || result.val === undefined) { + return undefined; + } + + return result.val; + } catch (e) { + recordPlatformFailure("getBillingLimit", "caught"); + return undefined; + } +} + +export async function setBillingLimit( + organizationId: string, + config: UpdateBillingLimitRequest, +): Promise { + if (!client) return undefined; + + const response = await client.fetch( + `/api/v1/orgs/${organizationId}/billing-limit`, + asPlatformSchema(BillingLimitResultSchema), + { + method: "PUT", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(config), + }, + ); + + if (!response.success) { + recordPlatformFailure("setBillingLimit", "no_success"); + throw new Error(response.error ?? "Error setting billing limit"); + } + + invalidateBillingLimitCaches(organizationId); + return response; +} + +export async function resolveBillingLimit( + organizationId: string, + payload: ResolveBillingLimitRequest, +): Promise { + if (!client) return undefined; + + const response = await client.fetch( + `/api/v1/orgs/${organizationId}/billing-limit/resolve`, + asPlatformSchema(BillingLimitResultSchema), + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }, + ); + + if (!response.success) { + recordPlatformFailure("resolveBillingLimit", "no_success"); + throw new Error(response.error ?? "Error resolving billing limit"); + } + + invalidateBillingLimitCaches(organizationId); + return response; +} + +/** Admin: orgs currently in grace or rejected — used by reconciliation worker (Phase 2). */ +export async function getActiveBillingLimits(): Promise { + if (!client) return undefined; + + try { + const response = await client.fetch( + `/api/v1/billing-limits/active`, + asPlatformSchema(BillingLimitsActiveResultSchema), + ); + if (!response.success) { + recordPlatformFailure("getActiveBillingLimits", "no_success"); + return undefined; + } + return response; + } catch (e) { + recordPlatformFailure("getActiveBillingLimits", "caught"); + return undefined; + } +} + +/** Admin: orgs with pending resolve side effects — used by reconciliation worker. */ +export async function getPendingBillingLimitResolves(): Promise< + BillingLimitsPendingResolvesResult | undefined +> { + if (!client) return undefined; + + try { + const response = await client.fetch( + `/api/v1/billing-limits/pending-resolves`, + asPlatformSchema(BillingLimitsPendingResolvesResultSchema), + ); + if (!response.success) { + recordPlatformFailure("getPendingBillingLimitResolves", "no_success"); + return undefined; + } + return response; + } catch (e) { + recordPlatformFailure("getPendingBillingLimitResolves", "caught"); + return undefined; + } +} + +/** Admin: mark billing limit resolve side effects as completed after webapp convergence. */ +export async function completeBillingLimitResolve( + organizationId: string, +): Promise<{ completed: boolean } | undefined> { + if (!client) return undefined; + + const response = await client.fetch( + `/api/v1/orgs/${organizationId}/billing-limit/resolve-complete`, + asPlatformSchema(z.object({ completed: z.boolean() })), + { + method: "POST", + }, + ); + + if (!response.success) { + recordPlatformFailure("completeBillingLimitResolve", "no_success"); + throw new Error(response.error ?? "Error completing billing limit resolve"); + } + + return response; +} + export async function getBillingAlerts( - organizationId: string + organizationId: string, ): Promise { if (!client) return undefined; const result = await client.getBillingAlerts(organizationId); @@ -651,7 +839,7 @@ export async function getBillingAlerts( export async function setBillingAlert( organizationId: string, - alert: UpdateBillingAlertsRequest + alert: UpdateBillingAlertsRequest, ): Promise { if (!client) return undefined; const result = await client.updateBillingAlerts(organizationId, alert); @@ -664,7 +852,7 @@ export async function setBillingAlert( export async function generateRegistryCredentials( projectId: string, - region: "us-east-1" | "eu-central-1" + region: "us-east-1" | "eu-central-1", ) { if (!client) return undefined; const result = await client.generateRegistryCredentials(projectId, region); @@ -683,7 +871,7 @@ export async function enqueueBuild( options: { skipPromotion?: boolean; configFilePath?: string; - } + }, ) { if (!client) return undefined; const result = await client.enqueueBuild(projectId, { deploymentId, artifactKey, options }); @@ -696,7 +884,7 @@ export async function enqueueBuild( } export async function getPrivateLinks( - organizationId: string + organizationId: string, ): Promise { if (!client) return undefined; @@ -717,7 +905,7 @@ export async function getPrivateLinks( export async function createPrivateLink( organizationId: string, - body: CreatePrivateLinkConnectionBody + body: CreatePrivateLinkConnectionBody, ): Promise { if (!client) throw new Error("Platform client not configured"); @@ -738,7 +926,7 @@ export async function createPrivateLink( export async function deletePrivateLink( organizationId: string, - connectionId: string + connectionId: string, ): Promise { if (!client) throw new Error("Platform client not configured"); @@ -756,7 +944,7 @@ export async function deletePrivateLink( } export async function getPrivateLinkRegions( - organizationId: string + organizationId: string, ): Promise { if (!client) return undefined; @@ -777,7 +965,7 @@ export async function getPrivateLinkRegions( export async function triggerInitialDeployment( projectId: string, - options: { environment: "preview" | "prod" | "staging" } + options: { environment: "preview" | "prod" | "staging" }, ): Promise { if (!client) return; @@ -801,6 +989,17 @@ export async function triggerInitialDeployment( } } +export type { + BillingLimitConfig, + BillingLimitPageData, + BillingLimitResult, + BillingLimitState, + BillingLimitsActiveResult, + EntitlementResult, + ResolveBillingLimitRequest, + UpdateBillingLimitRequest, +} from "~/services/billingLimit.schemas"; + export function isCloud(): boolean { const acceptableHosts = [ "https://cloud.trigger.dev", diff --git a/apps/webapp/app/services/routeBuilders/permissions.server.ts b/apps/webapp/app/services/routeBuilders/permissions.server.ts index 8d574abcea..4c8f372e3c 100644 --- a/apps/webapp/app/services/routeBuilders/permissions.server.ts +++ b/apps/webapp/app/services/routeBuilders/permissions.server.ts @@ -19,9 +19,13 @@ export type PermissionCheck = * returned booleans are display-only: the route builder's `authorization` * block is the real security boundary. */ +export function canManageBilling(ability: RbacAbility): boolean { + return ability.can("manage", { type: "billing" }); +} + export function checkPermissions( ability: RbacAbility, - checks: Record + checks: Record, ): Record { const result = {} as Record; for (const key in checks) { diff --git a/apps/webapp/app/services/upsertBranch.server.ts b/apps/webapp/app/services/upsertBranch.server.ts index 15042cae01..0583a67a05 100644 --- a/apps/webapp/app/services/upsertBranch.server.ts +++ b/apps/webapp/app/services/upsertBranch.server.ts @@ -14,6 +14,10 @@ import { getCurrentPlan, getLimit } from "./platform.v3.server"; import { type z } from "zod"; import invariant from "tiny-invariant"; import { type CreateBranchOptions } from "~/utils/branches"; +import { + applyBillingLimitPauseAfterEnvCreate, + getInitialEnvPauseStateForBillingLimit, +} from "~/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server"; type CreateBranchOptions = z.infer; @@ -32,7 +36,7 @@ export class UpsertBranchService { orgFilter: | { type: "userMembership"; userId: string } | { type: "orgId"; organizationId: string }, - { projectId, env, branchName, git }: CreateBranchOptions + { projectId, env, branchName, git }: CreateBranchOptions, ) { const parentEnvType = toBranchableEnvironmentType(env); // Dev branch creation is always user-scoped (org tokens are rejected upstream), @@ -138,6 +142,10 @@ export class UpsertBranchService { const apiKey = createApiKeyForEnv(parentEnvironment.type); const pkApiKey = createPkApiKeyForEnv(parentEnvironment.type); const shortcode = branchSlug; + const billingPause = await getInitialEnvPauseStateForBillingLimit( + parentEnvironment.organization.id, + parentEnvironment.type, + ); const now = new Date(); const branch = await this.#prismaClient.runtimeEnvironment.upsert({ @@ -153,6 +161,8 @@ export class UpsertBranchService { pkApiKey, shortcode, maximumConcurrencyLimit: parentEnvironment.maximumConcurrencyLimit, + paused: billingPause.paused, + pauseSource: billingPause.pauseSource, organization: { connect: { id: parentEnvironment.organization.id, @@ -174,9 +184,14 @@ export class UpsertBranchService { update: { git: git ?? undefined, }, + include: { + organization: true, + project: true, + }, }); const alreadyExisted = branch.createdAt < now; + await applyBillingLimitPauseAfterEnvCreate(branch); return { success: true as const, diff --git a/apps/webapp/app/utils/environmentPauseSource.ts b/apps/webapp/app/utils/environmentPauseSource.ts new file mode 100644 index 0000000000..c4da2ab296 --- /dev/null +++ b/apps/webapp/app/utils/environmentPauseSource.ts @@ -0,0 +1,2 @@ +/** Matches Prisma `EnvironmentPauseSource.BILLING_LIMIT`. Safe for client and server bundles. */ +export const ENVIRONMENT_PAUSE_SOURCE_BILLING_LIMIT = "BILLING_LIMIT" as const; diff --git a/apps/webapp/app/utils/pathBuilder.ts b/apps/webapp/app/utils/pathBuilder.ts index 3b65fde139..d0143301d9 100644 --- a/apps/webapp/app/utils/pathBuilder.ts +++ b/apps/webapp/app/utils/pathBuilder.ts @@ -173,7 +173,7 @@ export function v3ProjectPath(organization: OrgForPath, project: ProjectForPath) export function githubAppInstallPath(organizationSlug: string, redirectTo: string) { return `/github/install?org_slug=${organizationSlug}&redirect_to=${encodeURIComponent( - redirectTo + redirectTo, )}`; } @@ -188,7 +188,7 @@ export function vercelCallbackPath() { export function vercelResourcePath( organizationSlug: string, projectSlug: string, - environmentSlug: string + environmentSlug: string, ) { return `/resources/orgs/${organizationSlug}/projects/${projectSlug}/env/${environmentSlug}/vercel`; } @@ -196,17 +196,17 @@ export function vercelResourcePath( export function v3EnvironmentPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `/orgs/${organizationParam(organization)}/projects/${projectParam( - project + project, )}/env/${environmentParam(environment)}`; } export function v3TasksDashboardPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/tasks/dashboard`; } @@ -214,7 +214,7 @@ export function v3TasksDashboardPath( export function v3TasksStreamingPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/tasks/stream`; } @@ -222,7 +222,7 @@ export function v3TasksStreamingPath( export function v3ApiKeysPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/apikeys`; } @@ -230,7 +230,7 @@ export function v3ApiKeysPath( export function v3BulkActionsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/bulk-actions`; } @@ -239,7 +239,7 @@ export function v3BulkActionPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - bulkAction: { friendlyId: string } + bulkAction: { friendlyId: string }, ) { return `${v3BulkActionsPath(organization, project, environment)}/${bulkAction.friendlyId}`; } @@ -247,7 +247,7 @@ export function v3BulkActionPath( export function v3EnvironmentVariablesPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/environment-variables`; } @@ -255,7 +255,7 @@ export function v3EnvironmentVariablesPath( export function v3NewEnvironmentVariablesPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentVariablesPath(organization, project, environment)}/new`; } @@ -263,7 +263,7 @@ export function v3NewEnvironmentVariablesPath( export function v3ProjectAlertsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/alerts`; } @@ -271,7 +271,7 @@ export function v3ProjectAlertsPath( export function v3NewProjectAlertPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ProjectAlertsPath(organization, project, environment)}/new`; } @@ -279,7 +279,7 @@ export function v3NewProjectAlertPath( export function v3NewProjectAlertPathConnectToSlackPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ProjectAlertsPath(organization, project, environment)}/new/connect-to-slack`; } @@ -287,7 +287,7 @@ export function v3NewProjectAlertPathConnectToSlackPath( export function v3TestPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/test`; } @@ -295,7 +295,7 @@ export function v3TestPath( export function queryPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/query`; } @@ -304,7 +304,7 @@ export function v3CustomDashboardPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - dashboard: { friendlyId: string } + dashboard: { friendlyId: string }, ) { return `${v3EnvironmentPath(organization, project, environment)}/dashboards/custom/${ dashboard.friendlyId @@ -315,7 +315,7 @@ export function v3BuiltInDashboardPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - key: string + key: string, ) { return `${v3EnvironmentPath(organization, project, environment)}/dashboards/${key}`; } @@ -323,7 +323,7 @@ export function v3BuiltInDashboardPath( export function v3DashboardsLandingPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/dashboards`; } @@ -332,17 +332,17 @@ export function v3TestTaskPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - task: TaskForPath + task: TaskForPath, ) { return `${v3TestPath(organization, project, environment)}/tasks/${encodeURIComponent( - task.taskIdentifier + task.taskIdentifier, )}`; } export function v3PlaygroundPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/playground`; } @@ -351,7 +351,7 @@ export function v3PlaygroundAgentPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - agentSlug: string + agentSlug: string, ) { return `${v3PlaygroundPath(organization, project, environment)}/${encodeURIComponent(agentSlug)}`; } @@ -360,10 +360,10 @@ export function v3AgentTaskPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - agentSlug: string + agentSlug: string, ) { return `${v3EnvironmentPath(organization, project, environment)}/agents/${encodeURIComponent( - agentSlug + agentSlug, )}`; } @@ -371,10 +371,10 @@ export function v3StandardTaskPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - taskSlug: string + taskSlug: string, ) { return `${v3EnvironmentPath(organization, project, environment)}/tasks/standard/${encodeURIComponent( - taskSlug + taskSlug, )}`; } @@ -382,10 +382,10 @@ export function v3ScheduledTaskPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - taskSlug: string + taskSlug: string, ) { return `${v3EnvironmentPath(organization, project, environment)}/tasks/scheduled/${encodeURIComponent( - taskSlug + taskSlug, )}`; } @@ -393,7 +393,7 @@ export function v3RunsPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - filters?: TaskRunListSearchFilters + filters?: TaskRunListSearchFilters, ) { const searchParams = objectToSearchParams(filters); const query = searchParams ? `?${searchParams.toString()}` : ""; @@ -406,7 +406,7 @@ export function v3CreateBulkActionPath( environment: EnvironmentForPath, filters?: TaskRunListSearchFilters, mode?: "selected" | "filter", - action?: "replay" | "cancel" + action?: "replay" | "cancel", ) { const searchParams = objectToSearchParams(filters) ?? new URLSearchParams(); searchParams.set("bulkInspector", RUNS_BULK_INSPECTOR_OPEN_VALUE); @@ -425,7 +425,7 @@ export function v3RunPath( project: ProjectForPath, environment: EnvironmentForPath, run: v3RunForPath, - searchParams?: URLSearchParams + searchParams?: URLSearchParams, ) { const query = searchParams ? `?${searchParams.toString()}` : ""; return `${v3RunsPath(organization, project, environment)}/${run.friendlyId}${query}`; @@ -434,7 +434,7 @@ export function v3RunPath( export function v3RunRedirectPath( organization: OrgForPath, project: ProjectForPath, - run: v3RunForPath + run: v3RunForPath, ) { return `${v3ProjectPath(organization, project)}/runs/${run.friendlyId}`; } @@ -453,7 +453,7 @@ export function v3RunSpanPath( environment: EnvironmentForPath, run: v3RunForPath, span: v3SpanForPath, - searchParams?: URLSearchParams + searchParams?: URLSearchParams, ) { searchParams = searchParams ?? new URLSearchParams(); searchParams.set("span", span.spanId); @@ -464,7 +464,7 @@ export function v3RunStreamingPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - run: v3RunForPath + run: v3RunForPath, ) { return `${v3RunPath(organization, project, environment, run)}/stream`; } @@ -473,10 +473,10 @@ export function v3RunIdempotencyKeyResetPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - run: v3RunForPath + run: v3RunForPath, ) { return `/resources/orgs/${organizationParam(organization)}/projects/${projectParam( - project + project, )}/env/${environmentParam(environment)}/runs/${run.friendlyId}/idempotencyKey/reset`; } @@ -484,7 +484,7 @@ export function v3SchedulePath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - schedule: { friendlyId: string } + schedule: { friendlyId: string }, ) { return `${v3EnvironmentPath(organization, project, environment)}/schedules/${ schedule.friendlyId @@ -495,7 +495,7 @@ export function v3EditSchedulePath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - schedule: { friendlyId: string } + schedule: { friendlyId: string }, ) { return `${v3EnvironmentPath(organization, project, environment)}/schedules/edit/${ schedule.friendlyId @@ -505,7 +505,7 @@ export function v3EditSchedulePath( export function v3NewSchedulePath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/schedules/new`; } @@ -517,7 +517,7 @@ export function v3SchedulesAddOnPath(organization: OrgForPath) { export function v3QueuesPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/queues`; } @@ -526,7 +526,7 @@ export function v3WaitpointTokensPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - filters?: WaitpointSearchParams + filters?: WaitpointSearchParams, ) { const searchParams = objectToSearchParams(filters); const query = searchParams ? `?${searchParams.toString()}` : ""; @@ -538,7 +538,7 @@ export function v3WaitpointTokenPath( project: ProjectForPath, environment: EnvironmentForPath, token: { id: string }, - filters?: WaitpointSearchParams + filters?: WaitpointSearchParams, ) { const searchParams = objectToSearchParams(filters); const query = searchParams ? `?${searchParams.toString()}` : ""; @@ -548,7 +548,7 @@ export function v3WaitpointTokenPath( export function v3BatchesPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/batches`; } @@ -556,7 +556,7 @@ export function v3BatchesPath( export function v3SessionsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/sessions`; } @@ -565,7 +565,7 @@ export function v3SessionPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - session: { friendlyId: string } + session: { friendlyId: string }, ) { return `${v3SessionsPath(organization, project, environment)}/${session.friendlyId}`; } @@ -574,7 +574,7 @@ export function v3BatchPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - batch: { friendlyId: string } + batch: { friendlyId: string }, ) { return `${v3BatchesPath(organization, project, environment)}/${batch.friendlyId}`; } @@ -583,7 +583,7 @@ export function v3BatchRunsPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - batch: { friendlyId: string } + batch: { friendlyId: string }, ) { return `${v3RunsPath(organization, project, environment, { batchId: batch.friendlyId })}`; } @@ -591,7 +591,7 @@ export function v3BatchRunsPath( export function v3ProjectSettingsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/settings`; } @@ -599,7 +599,7 @@ export function v3ProjectSettingsPath( export function v3ProjectSettingsGeneralPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ProjectSettingsPath(organization, project, environment)}/general`; } @@ -607,7 +607,7 @@ export function v3ProjectSettingsGeneralPath( export function v3ProjectSettingsIntegrationsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ProjectSettingsPath(organization, project, environment)}/integrations`; } @@ -615,7 +615,7 @@ export function v3ProjectSettingsIntegrationsPath( export function v3LogsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/logs`; } @@ -623,7 +623,7 @@ export function v3LogsPath( export function v3PromptsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/prompts`; } @@ -633,7 +633,7 @@ export function v3PromptPath( project: ProjectForPath, environment: EnvironmentForPath, promptSlug: string, - version?: string | number + version?: string | number, ) { const base = `${v3PromptsPath(organization, project, environment)}/${promptSlug}`; return version != null ? `${base}?version=${version}` : base; @@ -642,7 +642,7 @@ export function v3PromptPath( export function v3ModelsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/models`; } @@ -651,7 +651,7 @@ export function v3ModelDetailPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - modelId: string + modelId: string, ) { return `${v3ModelsPath(organization, project, environment)}/${modelId}`; } @@ -659,7 +659,7 @@ export function v3ModelDetailPath( export function v3ModelComparePath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ModelsPath(organization, project, environment)}/compare`; } @@ -667,7 +667,7 @@ export function v3ModelComparePath( export function v3ErrorsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/errors`; } @@ -675,7 +675,7 @@ export function v3ErrorsPath( export function v3ErrorsConnectToSlackPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3ErrorsPath(organization, project, environment)}/connect-to-slack`; } @@ -684,7 +684,7 @@ export function v3ErrorPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - error: { fingerprint: string } + error: { fingerprint: string }, ) { return `${v3ErrorsPath(organization, project, environment)}/${error.fingerprint}`; } @@ -692,7 +692,7 @@ export function v3ErrorPath( export function v3DeploymentsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/deployments`; } @@ -702,7 +702,7 @@ export function v3DeploymentPath( project: ProjectForPath, environment: EnvironmentForPath, deployment: DeploymentForPath, - currentPage: number + currentPage: number, ) { const query = currentPage ? `?page=${currentPage}` : ""; return `${v3DeploymentsPath(organization, project, environment)}/${deployment.shortCode}${query}`; @@ -712,7 +712,7 @@ export function v3DeploymentVersionPath( organization: OrgForPath, project: ProjectForPath, environment: EnvironmentForPath, - version: string + version: string, ) { return `${v3DeploymentsPath(organization, project, environment)}?version=${version}`; } @@ -720,7 +720,7 @@ export function v3DeploymentVersionPath( export function branchesPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/branches`; } @@ -728,16 +728,15 @@ export function branchesPath( export function branchesDevPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/dev-branches`; } - export function concurrencyPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/concurrency`; } @@ -745,7 +744,7 @@ export function concurrencyPath( export function limitsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/limits`; } @@ -753,7 +752,7 @@ export function limitsPath( export function regionsPath( organization: OrgForPath, project: ProjectForPath, - environment: EnvironmentForPath + environment: EnvironmentForPath, ) { return `${v3EnvironmentPath(organization, project, environment)}/regions`; } @@ -764,8 +763,13 @@ export function v3BillingPath(organization: OrgForPath, message?: string) { }`; } +export function v3BillingLimitsPath(organization: OrgForPath) { + return `${organizationPath(organization)}/settings/billing-limits`; +} + +/** @deprecated Use v3BillingLimitsPath — redirects from billing-alerts are preserved */ export function v3BillingAlertsPath(organization: OrgForPath) { - return `${organizationPath(organization)}/settings/billing-alerts`; + return v3BillingLimitsPath(organization); } export function v3PrivateConnectionsPath(organization: OrgForPath) { diff --git a/apps/webapp/app/v3/billingLimitWorker.server.ts b/apps/webapp/app/v3/billingLimitWorker.server.ts new file mode 100644 index 0000000000..e1a9c16c57 --- /dev/null +++ b/apps/webapp/app/v3/billingLimitWorker.server.ts @@ -0,0 +1,189 @@ +import { Logger } from "@trigger.dev/core/logger"; +import { Worker as RedisWorker } from "@trigger.dev/redis-worker"; +import { z } from "zod"; +import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; +import { singleton } from "~/utils/singleton"; +import { BillingLimitConvergeEnvironmentsService } from "./services/billingLimit/billingLimitConvergeEnvironmentsService.server"; +import type { BillingLimitConvergeTargetState } from "./services/billingLimit/billingLimitConstants"; +import { + buildBillingLimitInProgressCancelJobId, + buildBillingLimitResolveJobId, +} from "./services/billingLimit/billingLimitConstants"; +import { runBillingLimitCancelInProgressRuns } from "./services/billingLimit/billingLimitCancelInProgressRuns.server"; +import { runPendingBillingLimitResolves } from "./services/billingLimit/billingLimitPendingResolveCoordinator.server"; +import type { PendingBillingLimitResolve } from "./services/billingLimit/billingLimitPendingResolve.types"; + +function initializeWorker() { + const redisOptions = { + keyPrefix: "billing-limit:worker:", + host: env.BILLING_LIMIT_WORKER_REDIS_HOST, + port: env.BILLING_LIMIT_WORKER_REDIS_PORT, + username: env.BILLING_LIMIT_WORKER_REDIS_USERNAME, + password: env.BILLING_LIMIT_WORKER_REDIS_PASSWORD, + enableAutoPipelining: true, + ...(env.BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED === "true" ? {} : { tls: {} }), + }; + + logger.debug( + `👨‍🏭 Initializing billing limit worker at host ${env.BILLING_LIMIT_WORKER_REDIS_HOST}`, + ); + + const worker = new RedisWorker({ + name: "billing-limit-worker", + redisOptions, + catalog: { + "billingLimit.convergeEnvironments": { + schema: z.object({ + organizationId: z.string(), + targetState: z.enum(["grace", "rejected", "ok"]), + }), + visibilityTimeoutMs: 60_000 * 10, + retry: { + maxAttempts: 5, + }, + }, + "billingLimit.reconcileTick": { + schema: z.object({}), + visibilityTimeoutMs: 60_000 * 5, + retry: { + maxAttempts: 3, + }, + }, + "billingLimit.cancelInProgressRuns": { + schema: z.object({ + organizationId: z.string(), + hitAt: z.string(), + }), + visibilityTimeoutMs: 60_000 * 10, + retry: { + maxAttempts: 5, + }, + }, + "billingLimit.resolve": { + schema: z.object({ + organizationId: z.string(), + resumeMode: z.enum(["queue", "new_only"]), + resolvedAt: z.string(), + }), + visibilityTimeoutMs: 60_000 * 10, + retry: { + maxAttempts: 5, + }, + }, + }, + concurrency: { + workers: env.BILLING_LIMIT_WORKER_CONCURRENCY_WORKERS, + tasksPerWorker: env.BILLING_LIMIT_WORKER_CONCURRENCY_TASKS_PER_WORKER, + limit: env.BILLING_LIMIT_WORKER_CONCURRENCY_LIMIT, + }, + pollIntervalMs: env.BILLING_LIMIT_WORKER_POLL_INTERVAL, + immediatePollIntervalMs: env.BILLING_LIMIT_WORKER_IMMEDIATE_POLL_INTERVAL, + shutdownTimeoutMs: env.BILLING_LIMIT_WORKER_SHUTDOWN_TIMEOUT_MS, + logger: new Logger("BillingLimitWorker", env.BILLING_LIMIT_WORKER_LOG_LEVEL), + jobs: { + "billingLimit.convergeEnvironments": async ({ payload }) => { + await BillingLimitConvergeEnvironmentsService.runConverge(payload); + }, + "billingLimit.reconcileTick": async () => { + await BillingLimitConvergeEnvironmentsService.runReconcileTick(); + await scheduleBillingLimitReconcileTick(worker); + }, + "billingLimit.cancelInProgressRuns": async ({ payload }) => { + await runBillingLimitCancelInProgressRuns(payload.organizationId, payload.hitAt); + }, + "billingLimit.resolve": async ({ payload }) => { + await runPendingBillingLimitResolves([payload]); + }, + }, + }); + + return worker; +} + +declare global { + // eslint-disable-next-line no-var + var __billingLimitWorkerStarted__: boolean | undefined; +} + +/** + * Bootstraps the billing-limit redis worker on webapp startup. + * + * Constructed via the module singleton (for enqueue from webhooks); started + * here so `sideEffects: false` builds keep an explicit entry-point side + * effect — do not rely on a bare `import "~/v3/billingLimitWorker.server"`. + */ +export function initBillingLimitWorker( + opts: { + isEnabled?: () => boolean; + } = {}, +): void { + const isEnabled = opts.isEnabled ?? (() => env.BILLING_LIMIT_WORKER_ENABLED === "true"); + + if (!isEnabled()) { + return; + } + + if (global.__billingLimitWorkerStarted__) { + return; + } + + const worker = billingLimitWorker; + + logger.debug( + `👨‍🏭 Starting billing limit worker at host ${env.BILLING_LIMIT_WORKER_REDIS_HOST}, reconcileIntervalMs = ${env.BILLING_LIMIT_RECONCILE_INTERVAL_MS}`, + ); + try { + worker.start(); + global.__billingLimitWorkerStarted__ = true; + void scheduleBillingLimitReconcileTick(worker).catch((error) => { + logger.error("Failed to schedule initial billing-limit reconcile tick", { + error, + }); + }); + } catch (error) { + global.__billingLimitWorkerStarted__ = false; + throw error; + } +} + +async function scheduleBillingLimitReconcileTick(worker: ReturnType) { + await worker.enqueue({ + id: "billingLimit.reconcileTick", + job: "billingLimit.reconcileTick", + payload: {}, + availableAt: new Date(Date.now() + env.BILLING_LIMIT_RECONCILE_INTERVAL_MS), + }); +} + +export const billingLimitWorker = singleton("billingLimitWorker", initializeWorker); + +export async function enqueueBillingLimitConverge( + organizationId: string, + targetState: BillingLimitConvergeTargetState, +) { + return billingLimitWorker.enqueue({ + id: `billingLimit.converge:${organizationId}:${targetState}`, + job: "billingLimit.convergeEnvironments", + payload: { organizationId, targetState }, + }); +} + +export async function enqueueBillingLimitCancelInProgressRuns( + organizationId: string, + hitAt: string, +) { + return billingLimitWorker.enqueue({ + id: buildBillingLimitInProgressCancelJobId(organizationId, hitAt), + job: "billingLimit.cancelInProgressRuns", + payload: { organizationId, hitAt }, + }); +} + +export async function enqueueBillingLimitResolve(pending: PendingBillingLimitResolve) { + return billingLimitWorker.enqueue({ + id: buildBillingLimitResolveJobId(pending.organizationId, pending.resolvedAt), + job: "billingLimit.resolve", + payload: pending, + }); +} diff --git a/apps/webapp/app/v3/outOfEntitlementError.server.ts b/apps/webapp/app/v3/outOfEntitlementError.server.ts new file mode 100644 index 0000000000..b5142c2562 --- /dev/null +++ b/apps/webapp/app/v3/outOfEntitlementError.server.ts @@ -0,0 +1,6 @@ +export class OutOfEntitlementError extends Error { + constructor() { + super("You can't trigger a task because you have run out of credits."); + this.name = "OutOfEntitlementError"; + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/BillingLimitBulkCancelService.server.ts b/apps/webapp/app/v3/services/billingLimit/BillingLimitBulkCancelService.server.ts new file mode 100644 index 0000000000..bf9c5cdb68 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/BillingLimitBulkCancelService.server.ts @@ -0,0 +1,223 @@ +import { BulkActionId } from "@trigger.dev/core/v3/isomorphic"; +import { + BulkActionNotificationType, + BulkActionType, + Prisma, + type PrismaClient, + type TaskRunStatus, +} from "@trigger.dev/database"; +import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus"; +import { prisma } from "~/db.server"; +import type { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + countInProgressRunsForBillableEnvironment, + countQueuedRunsForBillableEnvironment, + createBillingLimitRunsRepository, + getBillableEnvironmentsForBillingLimit, +} from "./billingLimitQueuedRuns.server"; +import { BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS } from "./billingLimitConstants"; + +export const BILLING_LIMIT_RESOLVE_CANCEL_SOURCE = "billing_limit_resolve_new_only"; +export const BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE = "billing_limit_in_progress"; + +export class BillingLimitBulkCancelIncompleteError extends Error { + constructor(readonly bulkActionId: string) { + super(`Billing limit bulk cancel did not complete within time budget: ${bulkActionId}`); + this.name = "BillingLimitBulkCancelIncompleteError"; + } +} + +type BulkCancelSource = + | typeof BILLING_LIMIT_RESOLVE_CANCEL_SOURCE + | typeof BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE; + +export type BillingLimitBulkCancelDeps = { + prismaClient?: PrismaClient; + createRunsRepository?: (organizationId: string) => Promise; + enqueueProcessBulkAction?: (bulkActionId: string) => Promise; + processBulkActionToCompletion?: ( + bulkActionId: string, + options?: { deadline?: number }, + ) => Promise<{ completed: boolean }>; +}; + +function resolveBulkCancelDeps(deps?: BillingLimitBulkCancelDeps) { + return { + prismaClient: deps?.prismaClient ?? prisma, + createRunsRepository: deps?.createRunsRepository ?? createBillingLimitRunsRepository, + enqueueProcessBulkAction: + deps?.enqueueProcessBulkAction ?? + (async (bulkActionId: string) => { + // Imported dynamically for the same reason as BulkActionService below: + // commonWorker.server transitively loads marqs -> the + // TaskRunConcurrencyTracker singleton, which throws when REDIS_HOST/ + // REDIS_PORT are unset (e.g. the webapp unit-test CI job). + const { commonWorker } = await import("~/v3/commonWorker.server"); + await commonWorker.enqueue({ + id: `processBulkAction-${bulkActionId}`, + job: "processBulkAction", + payload: { bulkActionId }, + }); + }), + processBulkActionToCompletion: + deps?.processBulkActionToCompletion ?? + (async (bulkActionId: string, options?: { deadline?: number }) => { + // Imported dynamically so this module doesn't eagerly load BulkActionV2 -> + // CancelTaskRunService -> marqs -> the TaskRunConcurrencyTracker singleton, + // which throws when REDIS_HOST/REDIS_PORT are unset (e.g. the webapp + // unit-test CI job). + const { BulkActionService } = await import("~/v3/services/bulk/BulkActionV2.server"); + const service = new BulkActionService(); + return service.processToCompletion(bulkActionId, { deadline: options?.deadline }); + }), + }; +} + +export class BillingLimitBulkCancelService { + static async cancelQueuedRuns( + organizationId: string, + options?: { + dedupeKey?: string; + waitForCompletion?: boolean; + bulkCancelDeadline?: number; + }, + deps?: BillingLimitBulkCancelDeps, + ): Promise<{ bulkActionIds: string[] }> { + return this.cancelRunsForBillableEnvironments( + organizationId, + { + source: BILLING_LIMIT_RESOLVE_CANCEL_SOURCE, + statuses: [...QUEUED_STATUSES], + name: "Billing limit resolve — cancel queued runs", + countRuns: countQueuedRunsForBillableEnvironment, + dedupeKey: options?.dedupeKey, + waitForCompletion: options?.waitForCompletion, + bulkCancelDeadline: options?.bulkCancelDeadline, + }, + deps, + ); + } + + static async cancelInProgressRuns( + organizationId: string, + options: { hitAt: string }, + deps?: BillingLimitBulkCancelDeps, + ): Promise<{ bulkActionIds: string[] }> { + return this.cancelRunsForBillableEnvironments( + organizationId, + { + source: BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE, + statuses: [...RUNNING_STATUSES], + name: "Billing limit hit — cancel in-progress runs", + countRuns: countInProgressRunsForBillableEnvironment, + dedupeKey: options.hitAt, + }, + deps, + ); + } + + private static async cancelRunsForBillableEnvironments( + organizationId: string, + options: { + source: BulkCancelSource; + statuses: TaskRunStatus[]; + name: string; + countRuns: typeof countQueuedRunsForBillableEnvironment; + dedupeKey?: string; + waitForCompletion?: boolean; + bulkCancelDeadline?: number; + }, + deps?: BillingLimitBulkCancelDeps, + ): Promise<{ bulkActionIds: string[] }> { + const { + prismaClient, + createRunsRepository, + enqueueProcessBulkAction, + processBulkActionToCompletion, + } = resolveBulkCancelDeps(deps); + + const environments = await getBillableEnvironmentsForBillingLimit(organizationId, prismaClient); + + if (environments.length === 0) { + return { bulkActionIds: [] }; + } + + const runsRepository = await createRunsRepository(organizationId); + const bulkActionIds: string[] = []; + const bulkActionInternalIds: string[] = []; + + for (const environment of environments) { + if (options.dedupeKey) { + const existing = await prismaClient.bulkActionGroup.findFirst({ + where: { + environmentId: environment.id, + type: BulkActionType.CANCEL, + dedupeKey: options.dedupeKey, + }, + select: { id: true, friendlyId: true }, + }); + + if (existing) { + if (options.waitForCompletion) { + bulkActionInternalIds.push(existing.id); + } else { + await enqueueProcessBulkAction(existing.id); + } + bulkActionIds.push(existing.friendlyId); + continue; + } + } + + const count = await options.countRuns(runsRepository, organizationId, environment); + + if (count === 0) { + continue; + } + + const { id, friendlyId } = BulkActionId.generate(); + + await prismaClient.bulkActionGroup.create({ + data: { + id, + friendlyId, + projectId: environment.projectId, + environmentId: environment.id, + name: options.name, + type: BulkActionType.CANCEL, + dedupeKey: options.dedupeKey, + params: { + statuses: options.statuses, + finalizeRun: true, + source: options.source, + ...(options.dedupeKey ? { dedupeKey: options.dedupeKey } : {}), + } as Prisma.InputJsonValue, + queryName: "bulk_action_v1", + totalCount: count, + completionNotification: BulkActionNotificationType.NONE, + }, + }); + + if (options.waitForCompletion) { + bulkActionInternalIds.push(id); + } else { + await enqueueProcessBulkAction(id); + } + + bulkActionIds.push(friendlyId); + } + + if (options.waitForCompletion) { + const deadline = + options.bulkCancelDeadline ?? Date.now() + BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS; + + for (const bulkActionId of bulkActionInternalIds) { + const result = await processBulkActionToCompletion(bulkActionId, { deadline }); + if (!result.completed) { + throw new BillingLimitBulkCancelIncompleteError(bulkActionId); + } + } + } + + return { bulkActionIds }; + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitCancelInProgressRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitCancelInProgressRuns.server.ts new file mode 100644 index 0000000000..1ef9af5ae4 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitCancelInProgressRuns.server.ts @@ -0,0 +1,8 @@ +import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server"; + +export async function runBillingLimitCancelInProgressRuns( + organizationId: string, + hitAt: string, +): Promise<{ bulkActionIds: string[] }> { + return BillingLimitBulkCancelService.cancelInProgressRuns(organizationId, { hitAt }); +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts new file mode 100644 index 0000000000..06ea4aec64 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitConstants.ts @@ -0,0 +1,38 @@ +import type { RuntimeEnvironmentType } from "@trigger.dev/database"; + +export const BILLABLE_ENVIRONMENT_TYPES = [ + "PRODUCTION", + "STAGING", + "PREVIEW", +] as const satisfies RuntimeEnvironmentType[]; + +export type BillableEnvironmentType = (typeof BILLABLE_ENVIRONMENT_TYPES)[number]; + +export const BILLING_LIMIT_CONVERGE_BATCH_SIZE = 50; + +/** Inline bulk-cancel budget for billing limit resolve (worker visibility is 10 min). */ +export const BILLING_LIMIT_RESOLVE_BULK_CANCEL_BUDGET_MS = 8 * 60_000; + +export type BillingLimitConvergeTargetState = "grace" | "rejected" | "ok"; + +export function isBillableEnvironmentType(type: RuntimeEnvironmentType): boolean { + return (BILLABLE_ENVIRONMENT_TYPES as readonly RuntimeEnvironmentType[]).includes(type); +} + +export function buildBillingLimitResolveDedupeKey( + organizationId: string, + resolvedAt: string, +): string { + return `billing-limit-resolve:${organizationId}:${resolvedAt}`; +} + +export function buildBillingLimitResolveJobId(organizationId: string, resolvedAt: string): string { + return `billingLimit.resolve:${organizationId}:${resolvedAt}`; +} + +export function buildBillingLimitInProgressCancelJobId( + organizationId: string, + hitAt: string, +): string { + return `billingLimit.cancelInProgress:${organizationId}:${hitAt}`; +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironments.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironments.server.ts new file mode 100644 index 0000000000..a39a6a4fb4 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironments.server.ts @@ -0,0 +1,212 @@ +import { + EnvironmentPauseSource, + type Organization, + type PrismaClient, + type Project, + type RuntimeEnvironment, +} from "@trigger.dev/database"; +import { prisma } from "~/db.server"; +import { logger } from "~/services/logger.server"; +import { + BILLABLE_ENVIRONMENT_TYPES, + BILLING_LIMIT_CONVERGE_BATCH_SIZE, + type BillingLimitConvergeTargetState, +} from "./billingLimitConstants"; + +export type ConvergeOrgResult = { + paused: number; + unpaused: number; +}; + +type EnvironmentWithRelations = RuntimeEnvironment & { + organization: Organization; + project: Project; +}; + +type UpdateEnvConcurrency = ( + environment: EnvironmentWithRelations, + maximumConcurrencyLimit?: number, +) => Promise; + +export async function convergeBillingLimitEnvironmentsForOrg( + organizationId: string, + targetState: BillingLimitConvergeTargetState, + options?: { + batchSize?: number; + prismaClient?: PrismaClient; + updateConcurrency?: UpdateEnvConcurrency; + }, +): Promise { + const db = options?.prismaClient ?? prisma; + const batchSize = options?.batchSize ?? BILLING_LIMIT_CONVERGE_BATCH_SIZE; + // Imported dynamically so this module (reachable from upsertBranch.server.ts at + // module load) doesn't eagerly load runQueue.server -> marqs -> triggerTaskV1 -> + // the autoIncrementCounter singleton, which throws when REDIS_HOST/REDIS_PORT are + // unset (e.g. the webapp unit-test CI job). + const updateConcurrency = + options?.updateConcurrency ?? + (async (environment, maximumConcurrencyLimit) => { + const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server"); + return updateEnvConcurrencyLimits(environment, maximumConcurrencyLimit); + }); + + if (targetState === "ok") { + return unpauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency); + } + + return pauseBillingLimitEnvironments(organizationId, db, batchSize, updateConcurrency); +} + +async function pauseBillingLimitEnvironments( + organizationId: string, + db: PrismaClient, + batchSize: number, + updateConcurrency: UpdateEnvConcurrency, +): Promise { + let paused = 0; + let cursor: string | undefined; + + while (true) { + const environments = await db.runtimeEnvironment.findMany({ + where: { + organizationId, + type: { in: [...BILLABLE_ENVIRONMENT_TYPES] }, + paused: false, + }, + take: batchSize, + ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}), + orderBy: { id: "asc" }, + include: { + organization: true, + project: true, + }, + }); + + if (environments.length === 0) { + break; + } + + for (const environment of environments) { + await pauseEnvironmentForBillingLimit(environment, db, updateConcurrency); + paused++; + } + + cursor = environments[environments.length - 1]?.id; + if (environments.length < batchSize) { + break; + } + } + + logger.info("Billing limit converge paused environments", { + organizationId, + paused, + }); + + return { paused, unpaused: 0 }; +} + +async function unpauseBillingLimitEnvironments( + organizationId: string, + db: PrismaClient, + batchSize: number, + updateConcurrency: UpdateEnvConcurrency, +): Promise { + let unpaused = 0; + let cursor: string | undefined; + + while (true) { + const environments = await db.runtimeEnvironment.findMany({ + where: { + organizationId, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + take: batchSize, + ...(cursor ? { skip: 1, cursor: { id: cursor } } : {}), + orderBy: { id: "asc" }, + include: { + organization: true, + project: true, + }, + }); + + if (environments.length === 0) { + break; + } + + for (const environment of environments) { + await resumeEnvironmentFromBillingLimit(environment, db, updateConcurrency); + unpaused++; + } + + cursor = environments[environments.length - 1]?.id; + if (environments.length < batchSize) { + break; + } + } + + logger.info("Billing limit converge unpaused environments", { + organizationId, + unpaused, + }); + + return { paused: 0, unpaused }; +} + +async function pauseEnvironmentForBillingLimit( + environment: EnvironmentWithRelations, + db: PrismaClient, + updateConcurrency: UpdateEnvConcurrency, +) { + const updated = await db.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + include: { + organization: true, + project: true, + }, + }); + + try { + await updateConcurrency(updated, 0); + } catch (error) { + await db.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { paused: false, pauseSource: null }, + }); + throw error; + } +} + +async function resumeEnvironmentFromBillingLimit( + environment: EnvironmentWithRelations, + db: PrismaClient, + updateConcurrency: UpdateEnvConcurrency, +) { + const updated = await db.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { + paused: false, + pauseSource: null, + }, + include: { + organization: true, + project: true, + }, + }); + + try { + await updateConcurrency(updated); + } catch (error) { + await db.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + }); + throw error; + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server.ts new file mode 100644 index 0000000000..732fb88a6a --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeEnvironmentsService.server.ts @@ -0,0 +1,24 @@ +import { z } from "zod"; +import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server"; +import { runBillingLimitReconcileTick } from "./runBillingLimitReconcileTick.server"; +import { seedBillingLimitReconcileQueue } from "./billingLimitReconcileQueue.server"; + +const ConvergePayloadSchema = z.object({ + organizationId: z.string(), + targetState: z.enum(["grace", "rejected", "ok"]), +}); + +export class BillingLimitConvergeEnvironmentsService { + static async seedReconcileQueue(organizationId: string) { + await seedBillingLimitReconcileQueue(organizationId); + } + + static async runConverge(payload: z.infer) { + const parsed = ConvergePayloadSchema.parse(payload); + return convergeBillingLimitEnvironmentsForOrg(parsed.organizationId, parsed.targetState); + } + + static async runReconcileTick() { + await runBillingLimitReconcileTick(); + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeResolve.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeResolve.server.ts new file mode 100644 index 0000000000..909c7353ca --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitConvergeResolve.server.ts @@ -0,0 +1,33 @@ +import { bustBillingLimitCaches } from "~/services/platform.v3.server"; +import { logger } from "~/services/logger.server"; +import { BillingLimitBulkCancelService } from "./BillingLimitBulkCancelService.server"; +import { buildBillingLimitResolveDedupeKey } from "./billingLimitConstants"; +import { convergeBillingLimitEnvironmentsForOrg } from "./billingLimitConvergeEnvironments.server"; +import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types"; + +export type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types"; + +export async function convergeBillingLimitResolve( + pending: PendingBillingLimitResolve, +): Promise { + const { organizationId, resumeMode, resolvedAt } = pending; + + bustBillingLimitCaches(organizationId); + + if (resumeMode === "new_only") { + await BillingLimitBulkCancelService.cancelQueuedRuns(organizationId, { + dedupeKey: buildBillingLimitResolveDedupeKey(organizationId, resolvedAt), + waitForCompletion: true, + }); + } + + await convergeBillingLimitEnvironmentsForOrg(organizationId, "ok"); + + logger.info("Converged billing limit resolve", { + organizationId, + resumeMode, + resolvedAt, + }); +} + +export { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server"; diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitHit.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitHit.server.ts new file mode 100644 index 0000000000..7b2b1101c3 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitHit.server.ts @@ -0,0 +1,26 @@ +export type BillingLimitHitPayload = { + organizationId: string; + hitAt: string; + cancelInProgressRuns: boolean; +}; + +export type BillingLimitHitDeps = { + bustCaches: (organizationId: string) => void; + seedReconcileQueue: (organizationId: string) => Promise; + enqueueConverge: (organizationId: string, targetState: "grace") => Promise; + enqueueCancelInProgressRuns: (organizationId: string, hitAt: string) => Promise; +}; + +/** Process billing limit grace hit from the billing platform webhook. */ +export async function processBillingLimitHit( + payload: BillingLimitHitPayload, + deps: BillingLimitHitDeps, +): Promise { + deps.bustCaches(payload.organizationId); + await deps.seedReconcileQueue(payload.organizationId); + await deps.enqueueConverge(payload.organizationId, "grace"); + + if (payload.cancelInProgressRuns) { + await deps.enqueueCancelInProgressRuns(payload.organizationId, payload.hitAt); + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolve.types.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolve.types.ts new file mode 100644 index 0000000000..faaada4ce9 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolve.types.ts @@ -0,0 +1,5 @@ +export type PendingBillingLimitResolve = { + organizationId: string; + resumeMode: "queue" | "new_only"; + resolvedAt: string; +}; diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveCoordinator.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveCoordinator.server.ts new file mode 100644 index 0000000000..04abf722c1 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveCoordinator.server.ts @@ -0,0 +1,55 @@ +import { logger } from "~/services/logger.server"; +import { classifyPendingBillingLimitResolveConvergeFailure } from "./billingLimitPendingResolveFailure.server"; +import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types"; + +export type RunPendingBillingLimitResolveDeps = { + converge?: (pending: PendingBillingLimitResolve) => Promise; + complete?: (organizationId: string) => Promise<{ completed: boolean } | undefined>; +}; + +export async function runPendingBillingLimitResolves( + pendingResolves: PendingBillingLimitResolve[], + deps: RunPendingBillingLimitResolveDeps = {}, +): Promise> { + const converge = + deps.converge ?? + (await import("./billingLimitConvergeResolve.server")).convergeBillingLimitResolve; + const complete = + deps.complete ?? (await import("~/services/platform.v3.server")).completeBillingLimitResolve; + + const stillPendingOrgIds = new Set(); + + for (const pending of pendingResolves) { + try { + await converge(pending); + } catch (error) { + logger.error("Failed to converge pending billing limit resolve", { + failureClass: classifyPendingBillingLimitResolveConvergeFailure(pending.resumeMode), + error, + organizationId: pending.organizationId, + resumeMode: pending.resumeMode, + resolvedAt: pending.resolvedAt, + }); + stillPendingOrgIds.add(pending.organizationId); + continue; + } + + try { + const completion = await complete(pending.organizationId); + if (!completion || completion.completed !== true) { + throw new Error("Billing platform client unavailable"); + } + } catch (error) { + logger.error("Failed to ack pending billing limit resolve", { + failureClass: "ack-only", + error, + organizationId: pending.organizationId, + resumeMode: pending.resumeMode, + resolvedAt: pending.resolvedAt, + }); + stillPendingOrgIds.add(pending.organizationId); + } + } + + return stillPendingOrgIds; +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveFailure.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveFailure.server.ts new file mode 100644 index 0000000000..15c232ed24 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitPendingResolveFailure.server.ts @@ -0,0 +1,11 @@ +export type PendingBillingLimitResolveFailureClass = + | "cancel-failing" + | "converge-failing" + | "ack-only"; + +/** Used in converge logs to classify stuck pending resolves. */ +export function classifyPendingBillingLimitResolveConvergeFailure( + resumeMode: "queue" | "new_only", +): Exclude { + return resumeMode === "new_only" ? "cancel-failing" : "converge-failing"; +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts new file mode 100644 index 0000000000..3b508af066 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitQueuedRuns.server.ts @@ -0,0 +1,99 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import type { TaskRunStatus } from "@trigger.dev/database"; +import { QUEUED_STATUSES, RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus"; +import { prisma } from "~/db.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { BILLABLE_ENVIRONMENT_TYPES } from "./billingLimitConstants"; + +export type BillableEnvironmentRef = { + id: string; + projectId: string; +}; + +export async function getBillableEnvironmentsForBillingLimit( + organizationId: string, + prismaClient: PrismaClient = prisma, +): Promise { + return prismaClient.runtimeEnvironment.findMany({ + where: { + organizationId, + type: { in: [...BILLABLE_ENVIRONMENT_TYPES] }, + }, + select: { + id: true, + projectId: true, + }, + }); +} + +export async function createBillingLimitRunsRepository(organizationId: string) { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + organizationId, + "standard", + ); + + return new RunsRepository({ + clickhouse, + prisma: prisma as PrismaClient, + }); +} + +export async function countQueuedRunsForBillableEnvironment( + runsRepository: RunsRepository, + organizationId: string, + environment: BillableEnvironmentRef, +): Promise { + return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [ + ...QUEUED_STATUSES, + ]); +} + +export async function countInProgressRunsForBillableEnvironment( + runsRepository: RunsRepository, + organizationId: string, + environment: BillableEnvironmentRef, +): Promise { + return countRunsForBillableEnvironment(runsRepository, organizationId, environment, [ + ...RUNNING_STATUSES, + ]); +} + +async function countRunsForBillableEnvironment( + runsRepository: RunsRepository, + organizationId: string, + environment: BillableEnvironmentRef, + statuses: TaskRunStatus[], +): Promise { + return runsRepository.countRuns({ + organizationId, + projectId: environment.projectId, + environmentId: environment.id, + statuses, + }); +} + +/** Same source as BillingLimitBulkCancelService — ClickHouse countRuns(QUEUED_STATUSES). */ +export async function countBillableQueuedRunsForOrganization( + organizationId: string, +): Promise { + const environments = await getBillableEnvironmentsForBillingLimit(organizationId); + + if (environments.length === 0) { + return 0; + } + + const runsRepository = await createBillingLimitRunsRepository(organizationId); + + let total = 0; + + for (const environment of environments) { + total += await countQueuedRunsForBillableEnvironment( + runsRepository, + organizationId, + environment, + ); + } + + return total; +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileQueue.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileQueue.server.ts new file mode 100644 index 0000000000..1cb1d4a89e --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileQueue.server.ts @@ -0,0 +1,35 @@ +import { env } from "~/env.server"; +import { createRedisClient } from "~/redis.server"; +import { singleton } from "~/utils/singleton"; + +const RECONCILE_QUEUE_KEY = "billing-limit:reconcile-queue"; + +function createQueueRedis() { + return createRedisClient("billing-limit:reconcile", { + keyPrefix: "", + host: env.BILLING_LIMIT_WORKER_REDIS_HOST, + port: env.BILLING_LIMIT_WORKER_REDIS_PORT, + username: env.BILLING_LIMIT_WORKER_REDIS_USERNAME, + password: env.BILLING_LIMIT_WORKER_REDIS_PASSWORD, + tlsDisabled: env.BILLING_LIMIT_WORKER_REDIS_TLS_DISABLED === "true", + }); +} + +const queueRedis = singleton("billingLimitReconcileQueueRedis", createQueueRedis); + +export async function seedBillingLimitReconcileQueue(organizationId: string): Promise { + await queueRedis.sadd(RECONCILE_QUEUE_KEY, organizationId); +} + +export async function readBillingLimitReconcileQueue(): Promise { + return queueRedis.smembers(RECONCILE_QUEUE_KEY); +} + +export async function removeFromBillingLimitReconcileQueue( + organizationIds: string[], +): Promise { + if (organizationIds.length === 0) { + return; + } + await queueRedis.srem(RECONCILE_QUEUE_KEY, ...organizationIds); +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileTarget.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileTarget.server.ts new file mode 100644 index 0000000000..8c32478974 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitReconcileTarget.server.ts @@ -0,0 +1,20 @@ +import type { BillingLimitConvergeTargetState } from "./billingLimitConstants"; +import type { OrgReconcileTarget } from "./billingLimitReconciliation.server"; + +export async function reconcileBillingLimitTarget( + target: OrgReconcileTarget, + deps: { + bustCaches: (organizationId: string) => void; + enqueueConverge: ( + organizationId: string, + targetState: BillingLimitConvergeTargetState, + ) => Promise; + }, +) { + // Safety net when webhooks are lost: bust stale entitlement after reject or resolve. + if (target.targetState === "rejected" || target.targetState === "ok") { + deps.bustCaches(target.organizationId); + } + + await deps.enqueueConverge(target.organizationId, target.targetState); +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitReconciliation.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitReconciliation.server.ts new file mode 100644 index 0000000000..7362acde40 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitReconciliation.server.ts @@ -0,0 +1,119 @@ +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import { prisma } from "~/db.server"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { getActiveBillingLimits, getBillingLimit } from "~/services/platform.v3.server"; +import type { BillingLimitConvergeTargetState } from "./billingLimitConstants"; +import { + readBillingLimitReconcileQueue, + removeFromBillingLimitReconcileQueue, +} from "./billingLimitReconcileQueue.server"; + +export type OrgReconcileTarget = { + organizationId: string; + targetState: BillingLimitConvergeTargetState; +}; + +export function resolveConvergeTargetFromBillingLimit( + billingLimit: BillingLimitResult | undefined, +): BillingLimitConvergeTargetState { + if (!billingLimit?.isConfigured) { + return "ok"; + } + + if (billingLimit.limitState.status === "grace") { + return "grace"; + } + + if (billingLimit.limitState.status === "rejected") { + return "rejected"; + } + + return "ok"; +} + +export async function getOrgIdsWithBillingPauseSource(): Promise { + const rows = await prisma.runtimeEnvironment.findMany({ + where: { + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + select: { + organizationId: true, + }, + distinct: ["organizationId"], + }); + + return rows.map((row) => row.organizationId); +} + +export function collectOrgIdsNeedingBillingLimitLookup(options: { + staleOrgIds: string[]; + queuedOrgIds: string[]; + excludeOrgIds: Set; + coveredOrgIds: Set; +}): string[] { + const orgIds = new Set(); + + for (const organizationId of [...options.staleOrgIds, ...options.queuedOrgIds]) { + if (options.excludeOrgIds.has(organizationId) || options.coveredOrgIds.has(organizationId)) { + continue; + } + + orgIds.add(organizationId); + } + + return [...orgIds]; +} + +export async function collectOrgsToReconcile(options?: { excludeOrgIds?: Set }): Promise<{ + targets: OrgReconcileTarget[]; + queuedOrgIds: string[]; +}> { + const excludeOrgIds = options?.excludeOrgIds ?? new Set(); + const targetByOrgId = new Map(); + + const activeLimits = await getActiveBillingLimits(); + if (activeLimits) { + for (const org of activeLimits.orgs) { + if (excludeOrgIds.has(org.orgId)) { + continue; + } + targetByOrgId.set(org.orgId, org.limitState); + } + } + + const [staleOrgIds, queuedOrgIds] = await Promise.all([ + getOrgIdsWithBillingPauseSource(), + readBillingLimitReconcileQueue(), + ]); + + const orgIdsNeedingLookup = collectOrgIdsNeedingBillingLimitLookup({ + staleOrgIds, + queuedOrgIds, + excludeOrgIds, + coveredOrgIds: new Set(targetByOrgId.keys()), + }); + + await Promise.all( + orgIdsNeedingLookup.map(async (organizationId) => { + const billingLimit = await getBillingLimit(organizationId); + targetByOrgId.set(organizationId, resolveConvergeTargetFromBillingLimit(billingLimit)); + }), + ); + + return { + targets: Array.from(targetByOrgId.entries()).map(([organizationId, targetState]) => ({ + organizationId, + targetState, + })), + queuedOrgIds, + }; +} + +export async function clearProcessedReconcileQueueEntries( + queuedOrgIds: string[], + processedOrgIds: string[], +): Promise { + const processed = new Set(processedOrgIds); + const toRemove = queuedOrgIds.filter((orgId) => processed.has(orgId)); + await removeFromBillingLimitReconcileQueue(toRemove); +} diff --git a/apps/webapp/app/v3/services/billingLimit/billingLimitResolve.server.ts b/apps/webapp/app/v3/services/billingLimit/billingLimitResolve.server.ts new file mode 100644 index 0000000000..d45e51647e --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/billingLimitResolve.server.ts @@ -0,0 +1,15 @@ +import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types"; + +export type BillingLimitResolveDeps = { + bustCaches: (organizationId: string) => void; + enqueueResolve: (pending: PendingBillingLimitResolve) => Promise; +}; + +/** Process billing limit resolve from the billing platform webhook. */ +export async function processBillingLimitResolve( + pending: PendingBillingLimitResolve, + deps: BillingLimitResolveDeps, +): Promise { + deps.bustCaches(pending.organizationId); + await deps.enqueueResolve(pending); +} diff --git a/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts new file mode 100644 index 0000000000..2689a3dc9b --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/getBillingLimitQueuedRunCount.server.ts @@ -0,0 +1,16 @@ +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import { prisma } from "~/db.server"; +import { countBillableQueuedRunsForOrganization } from "./billingLimitQueuedRuns.server"; + +export async function getBillingLimitQueuedRunCount(organizationId: string): Promise { + return countBillableQueuedRunsForOrganization(organizationId); +} + +export async function countBillingLimitPausedEnvironments(organizationId: string): Promise { + return prisma.runtimeEnvironment.count({ + where: { + organizationId, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + }); +} diff --git a/apps/webapp/app/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server.ts b/apps/webapp/app/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server.ts new file mode 100644 index 0000000000..596c45867d --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server.ts @@ -0,0 +1,72 @@ +import { EnvironmentPauseSource, type RuntimeEnvironmentType } from "@trigger.dev/database"; +import type { Organization, Project, RuntimeEnvironment } from "@trigger.dev/database"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { logger } from "~/services/logger.server"; +import { isBillableEnvironmentType } from "./billingLimitConstants"; +import { resolveConvergeTargetFromBillingLimit } from "./billingLimitReconciliation.server"; + +export type InitialEnvPauseState = { + paused: boolean; + pauseSource: typeof EnvironmentPauseSource.BILLING_LIMIT | null; +}; + +export type GetInitialEnvPauseStateDeps = { + getBillingLimit?: (organizationId: string) => Promise; +}; + +export async function getInitialEnvPauseStateForBillingLimit( + organizationId: string, + type: RuntimeEnvironmentType, + deps: GetInitialEnvPauseStateDeps = {}, +): Promise { + if (!isBillableEnvironmentType(type)) { + return { paused: false, pauseSource: null }; + } + + let billingLimit: BillingLimitResult | undefined; + try { + billingLimit = deps.getBillingLimit + ? await deps.getBillingLimit(organizationId) + : await (await import("~/services/platform.v3.server")).getBillingLimit(organizationId); + } catch (error) { + logger.error("Failed to fetch billing limit for initial env pause state", { + organizationId, + error, + }); + return { paused: false, pauseSource: null }; + } + + const targetState = resolveConvergeTargetFromBillingLimit(billingLimit); + + if (targetState === "grace" || targetState === "rejected") { + return { + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }; + } + + return { paused: false, pauseSource: null }; +} + +export async function applyBillingLimitPauseAfterEnvCreate( + environment: RuntimeEnvironment & { organization: Organization; project: Project }, +): Promise { + if (!environment.paused || environment.pauseSource !== EnvironmentPauseSource.BILLING_LIMIT) { + return; + } + + try { + // Imported dynamically so this module (pulled in at module load by + // upsertBranch.server.ts) doesn't eagerly load runQueue.server -> marqs -> + // triggerTaskV1 -> the autoIncrementCounter singleton, which throws when + // REDIS_HOST/REDIS_PORT are unset (e.g. the webapp unit-test CI job). + const { updateEnvConcurrencyLimits } = await import("~/v3/runQueue.server"); + await updateEnvConcurrencyLimits(environment, 0); + } catch (error) { + logger.error("Failed to apply billing-limit pause after env create", { + environmentId: environment.id, + organizationId: environment.organizationId, + error, + }); + } +} diff --git a/apps/webapp/app/v3/services/billingLimit/manualPauseEnvironmentGuard.server.ts b/apps/webapp/app/v3/services/billingLimit/manualPauseEnvironmentGuard.server.ts new file mode 100644 index 0000000000..a1026349b2 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/manualPauseEnvironmentGuard.server.ts @@ -0,0 +1,44 @@ +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import type { PauseStatus } from "~/v3/services/pauseEnvironment.server"; + +/** + * Guards manual pause/resume API calls while an environment is billing-paused. + * + * Design trade-off: billing-limit converge unpauses every environment with + * `pauseSource=BILLING_LIMIT` on resolve. We therefore do not record a + * separate manual pause on top of billing enforcement — a manual pause attempt + * while already billing-paused is a silent no-op (`success: true`, still + * paused). If the limit is later resolved, that environment is unpaused with + * the rest, even if the caller intended to keep it paused. + * + * The queues UI hides pause/resume while `pauseSource=BILLING_LIMIT`; API + * callers can still hit this path and should treat the no-op as idempotent. + */ +export function getManualPauseEnvironmentResult( + action: PauseStatus, + pauseSource: EnvironmentPauseSource | null | undefined, +): + | { proceed: true } + | { proceed: false; success: true; state: PauseStatus } + | { proceed: false; success: false; error: string } { + if (action === "resumed" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) { + return { + proceed: false, + success: false, + error: + "This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.", + }; + } + + if (action === "paused" && pauseSource === EnvironmentPauseSource.BILLING_LIMIT) { + // Already billing-paused; do not overwrite pauseSource so resolve converge + // can still find and unpause this environment. + return { + proceed: false, + success: true, + state: "paused", + }; + } + + return { proceed: true }; +} diff --git a/apps/webapp/app/v3/services/billingLimit/runBillingLimitReconcileTick.server.ts b/apps/webapp/app/v3/services/billingLimit/runBillingLimitReconcileTick.server.ts new file mode 100644 index 0000000000..ff0822b345 --- /dev/null +++ b/apps/webapp/app/v3/services/billingLimit/runBillingLimitReconcileTick.server.ts @@ -0,0 +1,75 @@ +import type { BillingLimitsPendingResolvesResult } from "~/services/billingLimit.schemas"; +import { logger } from "~/services/logger.server"; +import { runPendingBillingLimitResolves } from "./billingLimitPendingResolveCoordinator.server"; +import type { PendingBillingLimitResolve } from "./billingLimitPendingResolve.types"; +import type { OrgReconcileTarget } from "./billingLimitReconciliation.server"; +import type { reconcileBillingLimitTarget } from "./billingLimitReconcileTarget.server"; + +export type RunBillingLimitReconcileTickDeps = { + getPendingResolves?: () => Promise; + runPendingResolves?: (pendingResolves: PendingBillingLimitResolve[]) => Promise>; + collectOrgs?: (options?: { excludeOrgIds?: Set }) => Promise<{ + targets: OrgReconcileTarget[]; + queuedOrgIds: string[]; + }>; + reconcileTarget?: typeof reconcileBillingLimitTarget; + clearProcessedQueue?: (queuedOrgIds: string[], processedOrgIds: string[]) => Promise; + bustCaches?: (organizationId: string) => void; + enqueueConverge?: ( + organizationId: string, + targetState: OrgReconcileTarget["targetState"], + ) => Promise; +}; + +export async function runBillingLimitReconcileTick( + deps: RunBillingLimitReconcileTickDeps = {}, +): Promise { + const getPendingResolves = + deps.getPendingResolves ?? + (await import("~/services/platform.v3.server")).getPendingBillingLimitResolves; + const runPendingResolves = deps.runPendingResolves ?? runPendingBillingLimitResolves; + const collectOrgs = + deps.collectOrgs ?? + (await import("./billingLimitReconciliation.server")).collectOrgsToReconcile; + const reconcileTarget = + deps.reconcileTarget ?? + (await import("./billingLimitReconcileTarget.server")).reconcileBillingLimitTarget; + const clearProcessedQueue = + deps.clearProcessedQueue ?? + (await import("./billingLimitReconciliation.server")).clearProcessedReconcileQueueEntries; + const bustCaches = + deps.bustCaches ?? (await import("~/services/platform.v3.server")).bustBillingLimitCaches; + + const pendingResolves = (await getPendingResolves())?.orgs ?? []; + const stillPendingOrgIds = await runPendingResolves(pendingResolves); + + const { targets, queuedOrgIds } = await collectOrgs({ + excludeOrgIds: stillPendingOrgIds, + }); + + const enqueueConverge = + deps.enqueueConverge ?? + (async (organizationId, targetState) => { + const { enqueueBillingLimitConverge } = await import("~/v3/billingLimitWorker.server"); + await enqueueBillingLimitConverge(organizationId, targetState); + }); + + const processedOrgIds: string[] = []; + for (const target of targets) { + try { + await reconcileTarget(target, { + bustCaches, + enqueueConverge, + }); + processedOrgIds.push(target.organizationId); + } catch (error) { + logger.error("Failed to reconcile billing limit target", { + organizationId: target.organizationId, + targetState: target.targetState, + error, + }); + } + } + + await clearProcessedQueue(queuedOrgIds, processedOrgIds); +} diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts index 76d550c700..4b360ed9b3 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts @@ -26,6 +26,15 @@ import { v3BulkActionPath } from "~/utils/pathBuilder"; import { formatDateTime } from "~/components/primitives/DateTime"; import pMap from "p-map"; +export type ProcessToCompletionOptions = { + /** Absolute timestamp (ms) after which processing stops and returns incomplete. */ + deadline?: number; +}; + +export type ProcessToCompletionResult = { + completed: boolean; +}; + export class BulkActionService extends BaseService { public async create( organizationId: string, @@ -33,12 +42,15 @@ export class BulkActionService extends BaseService { environmentId: string, userId: string, payload: CreateBulkActionPayload, - request: Request + request: Request, ) { const filters = await getFilters(payload, request); // Count the runs that will be affected by the bulk action - const clickhouse = await clickhouseFactory.getClickhouseForOrganization(organizationId, "standard"); + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + organizationId, + "standard", + ); const runsRepository = new RunsRepository({ clickhouse, prisma: this._replica as PrismaClient, @@ -85,7 +97,38 @@ export class BulkActionService extends BaseService { }; } - public async process(bulkActionId: string) { + public async processToCompletion( + bulkActionId: string, + options?: ProcessToCompletionOptions, + ): Promise { + while (true) { + if (options?.deadline !== undefined && Date.now() >= options.deadline) { + return { completed: false }; + } + + const group = await this._prisma.bulkActionGroup.findFirst({ + where: { id: bulkActionId }, + select: { status: true }, + }); + + if ( + !group || + group.status === BulkActionStatus.COMPLETED || + group.status === BulkActionStatus.ABORTED + ) { + return { completed: true }; + } + + await this.process(bulkActionId, { continueInline: true }); + } + } + + public async process( + bulkActionId: string, + options?: { + continueInline?: boolean; + }, + ) { // 1. Get the bulk action group const group = await this._prisma.bulkActionGroup.findFirst({ where: { id: bulkActionId }, @@ -148,7 +191,10 @@ export class BulkActionService extends BaseService { ...rawParams, }); - const clickhouse = await clickhouseFactory.getClickhouseForOrganization(group.project.organizationId, "standard"); + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + group.project.organizationId, + "standard", + ); const runsRepository = new RunsRepository({ clickhouse, prisma: this._replica as PrismaClient, @@ -199,7 +245,7 @@ export class BulkActionService extends BaseService { taskEventStore: true, }, }, - this._replica + this._replica, ); await pMap( @@ -210,7 +256,7 @@ export class BulkActionService extends BaseService { reason: `Bulk action ${group.friendlyId} cancelled run`, bulkActionId: bulkActionId, finalizeRun, - }) + }), ); if (error) { logger.error("Failed to cancel run", { @@ -228,7 +274,7 @@ export class BulkActionService extends BaseService { } } }, - { concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY } + { concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }, ); break; @@ -244,7 +290,7 @@ export class BulkActionService extends BaseService { }, }, }, - this._replica + this._replica, ); await pMap( @@ -254,7 +300,7 @@ export class BulkActionService extends BaseService { replayService.call(run, { bulkActionId: bulkActionId, triggerSource: "dashboard", - }) + }), ); if (error) { logger.error("Failed to replay run, error", { @@ -277,7 +323,7 @@ export class BulkActionService extends BaseService { } } }, - { concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY } + { concurrency: env.BULK_ACTION_SUBBATCH_CONCURRENCY }, ); break; } @@ -349,7 +395,7 @@ export class BulkActionService extends BaseService { }, { friendlyId: group.friendlyId, - } + }, )}`, totalCount: updatedGroup.totalCount, successCount: updatedGroup.successCount, @@ -367,6 +413,10 @@ export class BulkActionService extends BaseService { } // 6. If there are more runs to process, queue the next batch + if (options?.continueInline) { + return; + } + await commonWorker.enqueue({ id: `processBulkAction-${bulkActionId}`, job: "processBulkAction", @@ -412,7 +462,7 @@ export class BulkActionService extends BaseService { async function getFilters( payload: CreateBulkActionPayload, - request: Request + request: Request, ): Promise { if (payload.mode === "selected") { return { diff --git a/apps/webapp/app/v3/services/pauseEnvironment.server.ts b/apps/webapp/app/v3/services/pauseEnvironment.server.ts index 99e588ca7d..aba0b0f6a9 100644 --- a/apps/webapp/app/v3/services/pauseEnvironment.server.ts +++ b/apps/webapp/app/v3/services/pauseEnvironment.server.ts @@ -1,6 +1,7 @@ -import { type PrismaClientOrTransaction } from "@trigger.dev/database"; +import { EnvironmentPauseSource, type PrismaClientOrTransaction } from "@trigger.dev/database"; import { prisma } from "~/db.server"; import { logger } from "~/services/logger.server"; +import { getManualPauseEnvironmentResult } from "~/v3/services/billingLimit/manualPauseEnvironmentGuard.server"; import { updateEnvConcurrencyLimits } from "../runQueue.server"; import { WithRunEngine } from "./baseService.server"; import { AuthenticatedEnvironment } from "~/services/apiAuth.server"; @@ -24,7 +25,7 @@ export class PauseEnvironmentService extends WithRunEngine { public async call( environment: AuthenticatedEnvironment, - action: PauseStatus + action: PauseStatus, ): Promise { try { const org = await this._prisma.organization.findFirst({ @@ -40,20 +41,56 @@ export class PauseEnvironmentService extends WithRunEngine { throw new Error("Organization not found"); } + const runtimeEnvironment = await this._prisma.runtimeEnvironment.findFirst({ + where: { id: environment.id }, + select: { + pauseSource: true, + }, + }); + + const manualPauseGuard = getManualPauseEnvironmentResult( + action, + runtimeEnvironment?.pauseSource, + ); + if (!manualPauseGuard.proceed) { + if (manualPauseGuard.success) { + return { + success: true, + state: manualPauseGuard.state, + }; + } + throw new Error(manualPauseGuard.error); + } + if (!org.runsEnabled && action === "resumed") { throw new Error( - "Runs are disabled for this organization. Your free plan has probably been exceeded. If not please contact support." + "Runs are disabled for this organization. Your free plan has probably been exceeded. If not please contact support.", ); } - await this._prisma.runtimeEnvironment.update({ - where: { - id: environment.id, - }, - data: { - paused: action === "paused", - }, - }); + if (action === "resumed") { + const resumed = await this._prisma.runtimeEnvironment.updateMany({ + where: { + id: environment.id, + NOT: { pauseSource: EnvironmentPauseSource.BILLING_LIMIT }, + }, + data: { + paused: false, + pauseSource: null, + }, + }); + + if (resumed.count === 0) { + throw new Error( + "This environment is paused because your organization reached its billing limit. Resolve the limit on the billing limits settings page to resume.", + ); + } + } else { + await this._prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { paused: true }, + }); + } if (action === "paused") { logger.debug("PauseEnvironmentService: pausing environment", { diff --git a/apps/webapp/app/v3/services/triggerTask.server.ts b/apps/webapp/app/v3/services/triggerTask.server.ts index 7bbaa0dd99..07330d7090 100644 --- a/apps/webapp/app/v3/services/triggerTask.server.ts +++ b/apps/webapp/app/v3/services/triggerTask.server.ts @@ -37,11 +37,7 @@ export type TriggerTaskServiceOptions = { triggerAction?: string; }; -export class OutOfEntitlementError extends Error { - constructor() { - super("You can't trigger a task because you have run out of credits."); - } -} +export { OutOfEntitlementError } from "../outOfEntitlementError.server"; export type TriggerTaskServiceResult = { run: TaskRun; @@ -64,7 +60,7 @@ export class TriggerTaskService extends WithRunEngine { environment: AuthenticatedEnvironment, body: TriggerTaskRequestBody, options: TriggerTaskServiceOptions = {}, - version?: RunEngineVersion + version?: RunEngineVersion, ): Promise { return await this.traceWithEnv("call()", environment, async (span) => { span.setAttribute("taskId", taskId); @@ -90,7 +86,7 @@ export class TriggerTaskService extends WithRunEngine { taskId: string, environment: AuthenticatedEnvironment, body: TriggerTaskRequestBody, - options: TriggerTaskServiceOptions = {} + options: TriggerTaskServiceOptions = {}, ): Promise { const service = new TriggerTaskServiceV1(this._prisma); return await service.call(taskId, environment, body, options); @@ -100,7 +96,7 @@ export class TriggerTaskService extends WithRunEngine { taskId: string, environment: AuthenticatedEnvironment, body: TriggerTaskRequestBody, - options: TriggerTaskServiceOptions = {} + options: TriggerTaskServiceOptions = {}, ): Promise { const traceEventConcern = new DefaultTraceEventsConcern(); @@ -113,7 +109,7 @@ export class TriggerTaskService extends WithRunEngine { idempotencyKeyConcern: new IdempotencyKeyConcern( this._prisma, this._engine, - traceEventConcern + traceEventConcern, ), traceEventConcern, tracer: tracer, diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 964c13c562..1ca170e22d 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -84,7 +84,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { // If we sent "shutdown", the worker will exit with code 0 after closing. code === 0 || worker.exitedAfterDisconnect; console.log( - `[cluster] worker ${worker.process.pid} exited (code=${code}, signal=${signal}, intentional=${intentional})` + `[cluster] worker ${worker.process.pid} exited (code=${code}, signal=${signal}, intentional=${intentional})`, ); // If it wasn't during a shutdown, replace the worker. if (!intentional) cluster.fork(); @@ -103,6 +103,10 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { // Remix fingerprints its assets so we can cache forever. app.use("/build", express.static("public/build", { immutable: true, maxAge: "1y" })); + // Stale dev builds can request an old hashed manifest; don't fall through to Remix. + app.use("/build", (_req, res) => { + res.status(404).end(); + }); // Everything else (like favicon.ico) is cached for an hour. You may want to be // more aggressive with this caching. @@ -116,7 +120,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { morgan("tiny", { skip: (_req, res) => suppressSuccessfulAccessLogs && res.statusCode >= 200 && res.statusCode < 300, - }) + }), ); process.title = ENABLE_CLUSTER @@ -165,7 +169,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { runWithHttpContext( { requestId, path: req.url, host: req.hostname, method: req.method, abortController }, - next + next, ); }); @@ -194,7 +198,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { createRequestHandler({ build, mode: MODE, - }) + }), ); } else { // we need to do the health check here at /healthcheck — forward @@ -207,7 +211,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { createRequestHandler({ build, mode: MODE, - }) + }), ); } @@ -215,7 +219,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { console.log( `✅ server ready: http://localhost:${port} [NODE_ENV: ${MODE}]${ ENABLE_CLUSTER && cluster.isWorker ? ` [worker ${cluster.worker?.id}/${process.pid}]` : "" - }` + }`, ); if (MODE === "development") { @@ -275,8 +279,8 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { // Setting the socket.destroy() error param causes an error event to be emitted which needs to be handled with socket.on("error") to prevent uncaught exceptions. socket.destroy( new Error( - "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request." - ) + "Cannot connect because of invalid path: Please include `/ws` in the path of your upgrade request.", + ), ); return; } diff --git a/apps/webapp/test/billingAlertsFormat.test.ts b/apps/webapp/test/billingAlertsFormat.test.ts new file mode 100644 index 0000000000..d366afb0b6 --- /dev/null +++ b/apps/webapp/test/billingAlertsFormat.test.ts @@ -0,0 +1,354 @@ +import { describe, expect, it } from "vitest"; +import { + clearedAlertsPayload, + emailsMatchSaved, + getAlertPreviewLimitCents, + getBillingLimitMode, + getConfiguredBillingLimitCents, + getUsageBarBillingLimitDollars, + hadSavedAlertsToClearOnLimitChange, + hasConfiguredAlerts, + hasLegacySpikeAlertLevels, + normalizeBillingAlertsFromApi, + percentageAlertLevelsToUiThresholds, + previewDollarAmountForPercent, + resetAlertsPayloadForLimitMode, + shouldClearAlertsOnLimitChange, + shouldResetAlertsOnLimitChange, + storedAlertsToThresholds, + thresholdsMatchSaved, + thresholdsToAlertPayload, + thresholdValuesAreUnique, + ABSOLUTE_ALERT_BASE_CENTS, +} from "~/components/billing/billingAlertsFormat"; + +const legacyDefaultLevels = [0.75, 0.9, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0]; + +describe("billingAlertsFormat", () => { + it("uses percentage thresholds saved in the new format", () => { + expect( + storedAlertsToThresholds( + { amount: 50, emails: [], alertLevels: [0.75, 0.9, 1.0] }, + "plan", + 5000, + 5000, + ), + ).toEqual([75, 90, 100]); + }); + + it("filters legacy spike multipliers above 100%", () => { + expect( + storedAlertsToThresholds( + { amount: 50, emails: [], alertLevels: legacyDefaultLevels }, + "plan", + 5000, + 5000, + ), + ).toEqual([75, 90, 100]); + + expect( + storedAlertsToThresholds( + { amount: 50, emails: [], alertLevels: legacyDefaultLevels }, + "none", + 5000, + 5000, + ), + ).toEqual([]); + }); + + it("reads legacy alerts saved against plan included usage", () => { + expect( + storedAlertsToThresholds( + { amount: 100, emails: [], alertLevels: [0.1, 0.5, 0.8, 2.0] }, + "plan", + 25_000, + 10_000, + ), + ).toEqual([10, 50, 80]); + + expect( + storedAlertsToThresholds( + { amount: 100, emails: [], alertLevels: [10, 50, 80, 200] }, + "plan", + 25_000, + 10_000, + ), + ).toEqual([10, 50, 80]); + + expect( + getAlertPreviewLimitCents({ amount: 100, emails: [], alertLevels: [] }, 25_000, 10_000), + ).toBe(10_000); + }); + + it("normalizes legacy API alerts with dollar amount field and whole percents", () => { + expect( + normalizeBillingAlertsFromApi({ + amount: 10_000, + emails: ["a@example.com"], + alertLevels: [10, 50, 80, 200], + }), + ).toEqual({ + amount: 100, + emails: ["a@example.com"], + alertLevels: [10, 50, 80, 200], + }); + + expect(percentageAlertLevelsToUiThresholds([10, 50, 80, 200])).toEqual([10, 50, 80]); + }); + + it("normalizes platform API alerts stored in cents", () => { + expect( + normalizeBillingAlertsFromApi({ + amount: 10_000, + emails: [], + alertLevels: [0.75, 0.9], + }), + ).toEqual({ + amount: 100, + emails: [], + alertLevels: [0.75, 0.9], + }); + }); + + it("normalizes cents-based alerts for billing limits under $10", () => { + const normalized = normalizeBillingAlertsFromApi({ + amount: 500, + emails: [], + alertLevels: [0.75, 0.9], + }); + + expect(normalized).toEqual({ + amount: 5, + emails: [], + alertLevels: [0.75, 0.9], + }); + + expect(storedAlertsToThresholds(normalized, "plan", 500, 500)).toEqual([75, 90]); + + expect(getAlertPreviewLimitCents(normalized, 500, 500)).toBe(500); + expect(previewDollarAmountForPercent(75, getAlertPreviewLimitCents(normalized, 500, 500))).toBe( + 3.75, + ); + }); + + it("returns no default thresholds when alerts are empty", () => { + expect( + storedAlertsToThresholds({ amount: 50, emails: [], alertLevels: [] }, "plan", 5000, 5000), + ).toEqual([]); + expect( + storedAlertsToThresholds({ amount: 1, emails: [], alertLevels: [] }, "none", 5000, 5000), + ).toEqual([]); + }); + + it("uses dollar thresholds for none mode with absolute base", () => { + expect( + storedAlertsToThresholds( + { amount: 1, emails: [], alertLevels: [100, 250] }, + "none", + 5000, + 5000, + ), + ).toEqual([100, 250]); + }); + + it("loads absolute dollar alerts after save with unlimited billing limit", () => { + const normalized = normalizeBillingAlertsFromApi({ + amount: ABSOLUTE_ALERT_BASE_CENTS, + emails: ["a@example.com"], + alertLevels: [100], + }); + + expect(normalized).toEqual({ + amount: 1, + emails: ["a@example.com"], + alertLevels: [100], + }); + + expect(storedAlertsToThresholds(normalized, "none", 5000, 5000)).toEqual([100]); + }); + + it("converts percentage UI values to API payload", () => { + expect(thresholdsToAlertPayload([75, 90], "plan", 5000)).toEqual({ + amount: 5000, + alertLevels: [0.75, 0.9], + }); + }); + + it("converts absolute UI values to API payload", () => { + expect(thresholdsToAlertPayload([100, 250], "none", 5000)).toEqual({ + amount: 100, + alertLevels: [100, 250], + }); + }); + + it("previews dollar amount from percentage and limit", () => { + expect(previewDollarAmountForPercent(75, 5000)).toBe(37.5); + expect(previewDollarAmountForPercent(10, 10_000)).toBe(10); + }); + + it("defaults unconfigured billing limit to none mode", () => { + expect(getBillingLimitMode({ isConfigured: false, gracePeriodMs: 86_400_000 })).toBe("none"); + }); + + it("detects configured alerts for the current billing limit mode", () => { + const billingLimit = { + isConfigured: true, + mode: "plan" as const, + effectiveAmountCents: 5000, + gracePeriodMs: 86_400_000, + }; + + expect( + hasConfiguredAlerts({ amount: 50, emails: [], alertLevels: [0.75, 0.9] }, billingLimit, 5000), + ).toBe(true); + + expect( + hasConfiguredAlerts({ amount: 50, emails: [], alertLevels: [] }, billingLimit, 5000), + ).toBe(false); + }); + + it("clears percentage alerts when switching from plan or custom to none", () => { + expect(shouldClearAlertsOnLimitChange("plan", "none")).toBe(true); + expect(shouldClearAlertsOnLimitChange("custom", "none")).toBe(true); + expect(shouldClearAlertsOnLimitChange("none", "none")).toBe(false); + expect(shouldClearAlertsOnLimitChange("plan", "custom")).toBe(false); + }); + + it("resets alerts when switching between percentage and dollar alert modes", () => { + expect(shouldResetAlertsOnLimitChange("none", "plan")).toBe(true); + expect(shouldResetAlertsOnLimitChange("none", "custom")).toBe(true); + expect(shouldResetAlertsOnLimitChange("plan", "none")).toBe(true); + expect(shouldResetAlertsOnLimitChange("plan", "custom")).toBe(false); + }); + + it("builds a cleared alerts payload for none mode", () => { + expect(clearedAlertsPayload(["a@example.com"])).toEqual({ + amount: 100, + alertLevels: [], + emails: ["a@example.com"], + }); + }); + + it("detects legacy spike alert levels above 100%", () => { + expect( + hasLegacySpikeAlertLevels( + { amount: 50, emails: [], alertLevels: [0.75, 0.9, 1.0, 2.0] }, + "plan", + 5000, + 5000, + ), + ).toBe(true); + + expect( + hasLegacySpikeAlertLevels( + { amount: 100, emails: [], alertLevels: [0.1, 0.5, 0.8, 2.0] }, + "plan", + 25_000, + 10_000, + ), + ).toBe(true); + + expect( + hasLegacySpikeAlertLevels( + { amount: 50, emails: [], alertLevels: [0.75, 0.9] }, + "plan", + 5000, + 5000, + ), + ).toBe(false); + + expect( + hasLegacySpikeAlertLevels( + { amount: 1, emails: [], alertLevels: [100, 250] }, + "none", + 5000, + 5000, + ), + ).toBe(false); + }); + + it("detects when saved alerts should be cleared on a limit format change", () => { + const billingLimit = { + isConfigured: true, + mode: "plan" as const, + effectiveAmountCents: 5000, + gracePeriodMs: 86_400_000, + }; + + expect( + hadSavedAlertsToClearOnLimitChange( + { amount: 50, emails: [], alertLevels: [0.75, 0.9] }, + billingLimit, + 5000, + ), + ).toBe(true); + + expect( + hadSavedAlertsToClearOnLimitChange( + { amount: 50, emails: ["a@example.com"], alertLevels: [] }, + billingLimit, + 5000, + ), + ).toBe(false); + }); + + it("compares threshold and email values for dirty form state", () => { + expect(thresholdsMatchSaved([90, 75], [75, 90])).toBe(true); + expect(thresholdsMatchSaved([75], [75, 90])).toBe(false); + expect(emailsMatchSaved(["a@example.com", ""], ["a@example.com"])).toBe(true); + expect(emailsMatchSaved(["b@example.com"], ["a@example.com"])).toBe(false); + }); + + it("detects duplicate alert thresholds", () => { + expect(thresholdValuesAreUnique([75, 90, 100])).toBe(true); + expect(thresholdValuesAreUnique([75, 75])).toBe(false); + expect(thresholdValuesAreUnique([100, 250, 100])).toBe(false); + }); + + it("returns configured billing limit cents for plan and custom modes", () => { + expect( + getConfiguredBillingLimitCents( + { + isConfigured: true, + mode: "custom", + amountCents: 25_000, + cancelInProgressRuns: false, + limitState: { status: "ok" }, + effectiveAmountCents: 25_000, + gracePeriodMs: 86_400_000, + }, + 5_000, + ), + ).toBe(25_000); + + expect( + getConfiguredBillingLimitCents( + { + isConfigured: true, + mode: "none", + cancelInProgressRuns: false, + limitState: { status: "ok" }, + effectiveAmountCents: null, + gracePeriodMs: 86_400_000, + }, + 5_000, + ), + ).toBeUndefined(); + }); + + it("maps usage bar billing limit dollars and hides when same as plan limit", () => { + const customLimit = { + isConfigured: true as const, + mode: "custom" as const, + amountCents: 25_000, + cancelInProgressRuns: false, + limitState: { status: "ok" as const }, + effectiveAmountCents: 25_000, + gracePeriodMs: 86_400_000, + }; + + expect(getUsageBarBillingLimitDollars(customLimit, 5_000)).toBe(250); + expect(getUsageBarBillingLimitDollars(customLimit, 25_000)).toBeUndefined(); + expect(getUsageBarBillingLimitDollars(undefined, 5_000)).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/billingLimit.schemas.test.ts b/apps/webapp/test/billingLimit.schemas.test.ts new file mode 100644 index 0000000000..597661cd6a --- /dev/null +++ b/apps/webapp/test/billingLimit.schemas.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, it } from "vitest"; +import { + BillingLimitResultSchema, + BillingLimitsPendingResolvesResultSchema, + EntitlementResultSchema, + ResolveBillingLimitRequestSchema, +} from "~/services/billingLimit.schemas"; + +describe("billingLimit.schemas", () => { + it("parses unconfigured billing limit", () => { + const result = BillingLimitResultSchema.parse({ + isConfigured: false, + gracePeriodMs: 86_400_000, + }); + + expect(result.isConfigured).toBe(false); + expect(result.gracePeriodMs).toBe(86_400_000); + }); + + it("parses configured mode none with limitState ok — not the same as unconfigured", () => { + const result = BillingLimitResultSchema.parse({ + isConfigured: true, + mode: "none", + cancelInProgressRuns: false, + effectiveAmountCents: null, + gracePeriodMs: 86_400_000, + limitState: { status: "ok" }, + }); + + expect(result.isConfigured).toBe(true); + if (result.isConfigured) { + expect(result.mode).toBe("none"); + expect(result.limitState.status).toBe("ok"); + expect(result.effectiveAmountCents).toBeNull(); + } + + // UI must use !isConfigured for the no-limit org banner — not mode === "none". + const unconfigured = BillingLimitResultSchema.parse({ + isConfigured: false, + gracePeriodMs: 86_400_000, + }); + expect(unconfigured.isConfigured).toBe(false); + expect(result.isConfigured).not.toBe(unconfigured.isConfigured); + }); + + it("parses configured billing limit in grace", () => { + const result = BillingLimitResultSchema.parse({ + isConfigured: true, + mode: "custom", + amountCents: 50_000, + cancelInProgressRuns: false, + effectiveAmountCents: 50_000, + gracePeriodMs: 86_400_000, + limitState: { + status: "grace", + hitAt: "2026-06-14T12:00:00.000Z", + graceEndsAt: "2026-06-15T12:00:00.000Z", + }, + }); + + expect(result.isConfigured).toBe(true); + if (result.isConfigured) { + expect(result.mode).toBe("custom"); + expect(result.limitState.status).toBe("grace"); + } + }); + + it("parses entitlement with billing_limit reason", () => { + const result = EntitlementResultSchema.parse({ + hasAccess: false, + reason: "billing_limit", + }); + + expect(result.hasAccess).toBe(false); + expect(result.reason).toBe("billing_limit"); + }); + + it("parses entitlement with free_tier_exceeded reason", () => { + const result = EntitlementResultSchema.parse({ + hasAccess: false, + reason: "free_tier_exceeded", + balance: 0, + usage: 100, + overage: 10, + }); + + expect(result.hasAccess).toBe(false); + expect(result.reason).toBe("free_tier_exceeded"); + }); + + it("parses entitlement with grace limit state", () => { + const result = EntitlementResultSchema.parse({ + hasAccess: true, + limitState: "grace", + }); + + expect(result.hasAccess).toBe(true); + expect(result.limitState).toBe("grace"); + }); + + it("parses resolve payload", () => { + const result = ResolveBillingLimitRequestSchema.parse({ + action: "increase", + newAmountCents: 150_000, + resumeMode: "queue", + }); + + expect(result.action).toBe("increase"); + expect(result.newAmountCents).toBe(150_000); + }); + + it("parses pending billing limit resolves from billing platform", () => { + const result = BillingLimitsPendingResolvesResultSchema.parse({ + orgs: [ + { + organizationId: "org_123", + resumeMode: "new_only", + resolvedAt: "2026-06-17T12:00:00.000Z", + }, + ], + }); + + expect(result.orgs).toHaveLength(1); + expect(result.orgs[0]?.resumeMode).toBe("new_only"); + }); +}); diff --git a/apps/webapp/test/billingLimitBulkCancelInProgress.test.ts b/apps/webapp/test/billingLimitBulkCancelInProgress.test.ts new file mode 100644 index 0000000000..bf93527d7d --- /dev/null +++ b/apps/webapp/test/billingLimitBulkCancelInProgress.test.ts @@ -0,0 +1,331 @@ +import { describe, expect, vi } from "vitest"; +import { setTimeout } from "node:timers/promises"; +import { postgresTest, replicationContainerTest } from "@internal/testcontainers"; +import { BulkActionType } from "@trigger.dev/database"; +import { + BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE, + BILLING_LIMIT_RESOLVE_CANCEL_SOURCE, + BillingLimitBulkCancelIncompleteError, + BillingLimitBulkCancelService, +} from "~/v3/services/billingLimit/BillingLimitBulkCancelService.server"; +import { countInProgressRunsForBillableEnvironment } from "~/v3/services/billingLimit/billingLimitQueuedRuns.server"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { + createRuntimeEnvironment, + createTestOrgProjectWithMember, + uniqueId, +} from "./fixtures/environmentVariablesFixtures"; +import { setupClickhouseReplication } from "./utils/replicationUtils"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("BillingLimitBulkCancelService.cancelQueuedRuns", () => { + postgresTest( + "processes bulk cancel inline when waitForCompletion is true", + async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const productionEnv = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + const dedupeKey = "billing-limit-resolve:org:2026-06-16T12:00:00.000Z"; + const enqueuedBulkActionIds: string[] = []; + const processedBulkActionIds: string[] = []; + + const result = await BillingLimitBulkCancelService.cancelQueuedRuns( + organization.id, + { dedupeKey, waitForCompletion: true }, + { + prismaClient: prisma, + createRunsRepository: async () => + ({ + countRuns: async () => 2, + }) as never, + enqueueProcessBulkAction: async (bulkActionId) => { + enqueuedBulkActionIds.push(bulkActionId); + }, + processBulkActionToCompletion: async (bulkActionId) => { + processedBulkActionIds.push(bulkActionId); + return { completed: true }; + }, + }, + ); + + expect(result.bulkActionIds).toHaveLength(1); + expect(enqueuedBulkActionIds).toEqual([]); + expect(processedBulkActionIds).toHaveLength(1); + + const group = await prisma.bulkActionGroup.findFirst({ + where: { + environmentId: productionEnv.id, + type: BulkActionType.CANCEL, + }, + }); + + expect(group?.name).toBe("Billing limit resolve — cancel queued runs"); + expect(group?.dedupeKey).toBe(dedupeKey); + expect(group?.params).toMatchObject({ + source: BILLING_LIMIT_RESOLVE_CANCEL_SOURCE, + dedupeKey, + finalizeRun: true, + }); + expect(processedBulkActionIds).toEqual([group?.id]); + }, + ); + + postgresTest( + "reuses existing bulk cancel and processes inline when waitForCompletion is true", + async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const productionEnv = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + const dedupeKey = "billing-limit-resolve:org:2026-06-16T12:00:00.000Z"; + + await prisma.bulkActionGroup.create({ + data: { + id: "bulk_existing_resolve", + friendlyId: "bulk_existing_resolve", + projectId: project.id, + environmentId: productionEnv.id, + name: "Existing resolve cancel", + type: BulkActionType.CANCEL, + dedupeKey, + params: { + statuses: ["PENDING"], + finalizeRun: true, + source: BILLING_LIMIT_RESOLVE_CANCEL_SOURCE, + dedupeKey, + }, + queryName: "bulk_action_v1", + totalCount: 1, + }, + }); + + const enqueuedBulkActionIds: string[] = []; + const processedBulkActionIds: string[] = []; + + const result = await BillingLimitBulkCancelService.cancelQueuedRuns( + organization.id, + { dedupeKey, waitForCompletion: true }, + { + prismaClient: prisma, + // Stubbed so the dedupe path doesn't build the default ClickHouse-backed + // repository, which queries the global $replica and hangs in the + // unit-test CI job (no reachable database/ClickHouse there). + createRunsRepository: async () => + ({ + countRuns: async () => 0, + }) as never, + enqueueProcessBulkAction: async (bulkActionId) => { + enqueuedBulkActionIds.push(bulkActionId); + }, + processBulkActionToCompletion: async (bulkActionId) => { + processedBulkActionIds.push(bulkActionId); + return { completed: true }; + }, + }, + ); + + expect(result.bulkActionIds).toEqual(["bulk_existing_resolve"]); + expect(enqueuedBulkActionIds).toEqual([]); + expect(processedBulkActionIds).toEqual(["bulk_existing_resolve"]); + }, + ); + + postgresTest("throws when inline bulk cancel exceeds the time budget", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + const dedupeKey = "billing-limit-resolve:org:2026-06-16T12:00:00.000Z"; + + await expect( + BillingLimitBulkCancelService.cancelQueuedRuns( + organization.id, + { dedupeKey, waitForCompletion: true, bulkCancelDeadline: Date.now() }, + { + prismaClient: prisma, + createRunsRepository: async () => + ({ + countRuns: async () => 2, + }) as never, + processBulkActionToCompletion: async () => ({ completed: false }), + }, + ), + ).rejects.toBeInstanceOf(BillingLimitBulkCancelIncompleteError); + }); +}); + +describe("BillingLimitBulkCancelService.cancelInProgressRuns", () => { + postgresTest("dedupes bulk cancel by hitAt per environment", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const productionEnv = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + const hitAt = "2026-06-16T12:00:00.000Z"; + + await prisma.bulkActionGroup.create({ + data: { + id: "bulk_existing", + friendlyId: "bulk_existing", + projectId: project.id, + environmentId: productionEnv.id, + name: "Existing in-progress cancel", + type: BulkActionType.CANCEL, + dedupeKey: hitAt, + params: { + statuses: ["EXECUTING"], + finalizeRun: true, + source: BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE, + dedupeKey: hitAt, + }, + queryName: "bulk_action_v1", + totalCount: 1, + }, + }); + + const enqueuedBulkActionIds: string[] = []; + + const result = await BillingLimitBulkCancelService.cancelInProgressRuns( + organization.id, + { hitAt }, + { + prismaClient: prisma, + // Stubbed so the dedupe path doesn't build the default ClickHouse-backed + // repository, which queries the global $replica and hangs in the + // unit-test CI job (no reachable database/ClickHouse there). + createRunsRepository: async () => + ({ + countRuns: async () => 0, + }) as never, + enqueueProcessBulkAction: async (bulkActionId) => { + enqueuedBulkActionIds.push(bulkActionId); + }, + }, + ); + + expect(result.bulkActionIds).toEqual(["bulk_existing"]); + expect(enqueuedBulkActionIds).toEqual(["bulk_existing"]); + + const groups = await prisma.bulkActionGroup.findMany({ + where: { environmentId: productionEnv.id, type: BulkActionType.CANCEL }, + }); + + expect(groups).toHaveLength(1); + }); + + replicationContainerTest( + "creates bulk cancel for in-progress runs in billable environments", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + const { clickhouse } = await setupClickhouseReplication({ + prisma, + databaseUrl: postgresContainer.getConnectionUri(), + clickhouseUrl: clickhouseContainer.getConnectionUrl(), + redisOptions, + }); + + const organization = await prisma.organization.create({ + data: { title: "billing-limit-in-progress-runs", slug: "billing-limit-in-progress-runs" }, + }); + + const project = await prisma.project.create({ + data: { + name: "billing-limit-in-progress-runs", + slug: "billing-limit-in-progress-runs", + organizationId: organization.id, + externalRef: "billing-limit-in-progress-runs", + }, + }); + + const productionEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: "prod", + pkApiKey: "prod", + shortcode: "prod", + }, + }); + + await prisma.taskRun.create({ + data: { + friendlyId: "run_executing_prod", + taskIdentifier: "running-task", + status: "EXECUTING", + payload: JSON.stringify({}), + traceId: "trace", + spanId: "span", + queue: "main", + runtimeEnvironmentId: productionEnv.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "PRODUCTION", + engine: "V2", + }, + }); + + await setTimeout(1000); + + const runsRepository = new RunsRepository({ prisma, clickhouse }); + + const count = await countInProgressRunsForBillableEnvironment( + runsRepository, + organization.id, + { id: productionEnv.id, projectId: project.id }, + ); + + expect(count).toBe(1); + + const hitAt = "2026-06-16T12:00:00.000Z"; + const enqueuedBulkActionIds: string[] = []; + + const result = await BillingLimitBulkCancelService.cancelInProgressRuns( + organization.id, + { hitAt }, + { + prismaClient: prisma, + createRunsRepository: async () => runsRepository, + enqueueProcessBulkAction: async (bulkActionId) => { + enqueuedBulkActionIds.push(bulkActionId); + }, + }, + ); + + expect(result.bulkActionIds).toHaveLength(1); + expect(enqueuedBulkActionIds).toHaveLength(1); + + const group = await prisma.bulkActionGroup.findFirst({ + where: { + environmentId: productionEnv.id, + type: BulkActionType.CANCEL, + }, + }); + + expect(group?.name).toBe("Billing limit hit — cancel in-progress runs"); + expect(group?.dedupeKey).toBe(hitAt); + expect(group?.params).toMatchObject({ + source: BILLING_LIMIT_IN_PROGRESS_CANCEL_SOURCE, + dedupeKey: hitAt, + finalizeRun: true, + }); + }, + ); +}); diff --git a/apps/webapp/test/billingLimitConvergeEnvironments.test.ts b/apps/webapp/test/billingLimitConvergeEnvironments.test.ts new file mode 100644 index 0000000000..0d2f0bb298 --- /dev/null +++ b/apps/webapp/test/billingLimitConvergeEnvironments.test.ts @@ -0,0 +1,107 @@ +import type { PrismaClient } from "@trigger.dev/database"; +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { postgresTest } from "@internal/testcontainers"; +import { convergeBillingLimitEnvironmentsForOrg } from "~/v3/services/billingLimit/billingLimitConvergeEnvironments.server"; +import { + createRuntimeEnvironment, + createTestOrgProjectWithMember, + uniqueId, +} from "./fixtures/environmentVariablesFixtures"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function createBillingPausedProductionEnv(prisma: PrismaClient) { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }, + }); + + return { organization, environment }; +} + +describe("convergeBillingLimitEnvironmentsForOrg", () => { + postgresTest("unpauses billable environments paused by billing limit", async ({ prisma }) => { + const { organization, environment } = await createBillingPausedProductionEnv(prisma); + + const result = await convergeBillingLimitEnvironmentsForOrg(organization.id, "ok", { + prismaClient: prisma, + updateConcurrency: async () => undefined, + }); + + expect(result).toEqual({ paused: 0, unpaused: 1 }); + + const envAfter = await prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { id: environment.id }, + }); + expect(envAfter.paused).toBe(false); + expect(envAfter.pauseSource).toBeNull(); + }); + + postgresTest("rolls back pause when concurrency update fails", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + await expect( + convergeBillingLimitEnvironmentsForOrg(organization.id, "grace", { + prismaClient: prisma, + updateConcurrency: async () => { + throw new Error("run queue unavailable"); + }, + }), + ).rejects.toThrow("run queue unavailable"); + + const envAfter = await prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { id: environment.id }, + }); + expect(envAfter.paused).toBe(false); + expect(envAfter.pauseSource).toBeNull(); + }); + + postgresTest("does not unpause environments paused for other reasons", async ({ prisma }) => { + const { organization, project } = await createTestOrgProjectWithMember(prisma); + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + slug: uniqueId("prod"), + }); + + await prisma.runtimeEnvironment.update({ + where: { id: environment.id }, + data: { + paused: true, + pauseSource: null, + }, + }); + + const result = await convergeBillingLimitEnvironmentsForOrg(organization.id, "ok", { + prismaClient: prisma, + updateConcurrency: async () => undefined, + }); + + expect(result).toEqual({ paused: 0, unpaused: 0 }); + + const envAfter = await prisma.runtimeEnvironment.findUniqueOrThrow({ + where: { id: environment.id }, + }); + expect(envAfter.paused).toBe(true); + expect(envAfter.pauseSource).toBeNull(); + }); +}); diff --git a/apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts b/apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts new file mode 100644 index 0000000000..b2c53faecb --- /dev/null +++ b/apps/webapp/test/billingLimitConvergeEnvironmentsService.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { reconcileBillingLimitTarget } from "~/v3/services/billingLimit/billingLimitReconcileTarget.server"; + +describe("reconcileBillingLimitTarget", () => { + it("busts billing limit caches for rejected targets before enqueueing converge", async () => { + const bustedOrgIds: string[] = []; + const enqueued: Array<{ organizationId: string; targetState: string }> = []; + + await reconcileBillingLimitTarget( + { organizationId: "org_123", targetState: "rejected" }, + { + bustCaches: (organizationId) => { + bustedOrgIds.push(organizationId); + }, + enqueueConverge: async (organizationId, targetState) => { + enqueued.push({ organizationId, targetState }); + }, + }, + ); + + expect(bustedOrgIds).toEqual(["org_123"]); + expect(enqueued).toEqual([{ organizationId: "org_123", targetState: "rejected" }]); + }); + + it("busts billing limit caches for ok targets before enqueueing converge", async () => { + const bustedOrgIds: string[] = []; + const enqueued: Array<{ organizationId: string; targetState: string }> = []; + + await reconcileBillingLimitTarget( + { organizationId: "org_123", targetState: "ok" }, + { + bustCaches: (organizationId) => { + bustedOrgIds.push(organizationId); + }, + enqueueConverge: async (organizationId, targetState) => { + enqueued.push({ organizationId, targetState }); + }, + }, + ); + + expect(bustedOrgIds).toEqual(["org_123"]); + expect(enqueued).toEqual([{ organizationId: "org_123", targetState: "ok" }]); + }); + + it("does not bust caches for grace targets", async () => { + const bustedOrgIds: string[] = []; + + await reconcileBillingLimitTarget( + { organizationId: "org_123", targetState: "grace" }, + { + bustCaches: (organizationId) => { + bustedOrgIds.push(organizationId); + }, + enqueueConverge: async () => undefined, + }, + ); + + expect(bustedOrgIds).toEqual([]); + }); +}); diff --git a/apps/webapp/test/billingLimitConvergeResolve.test.ts b/apps/webapp/test/billingLimitConvergeResolve.test.ts new file mode 100644 index 0000000000..565d566054 --- /dev/null +++ b/apps/webapp/test/billingLimitConvergeResolve.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { buildBillingLimitResolveDedupeKey } from "~/v3/services/billingLimit/billingLimitConstants"; +import { classifyPendingBillingLimitResolveConvergeFailure } from "~/v3/services/billingLimit/billingLimitPendingResolveFailure.server"; +import { runPendingBillingLimitResolves } from "~/v3/services/billingLimit/billingLimitPendingResolveCoordinator.server"; +import type { PendingBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitPendingResolve.types"; + +describe("billingLimitConvergeResolve", () => { + it("builds a stable dedupe key from org id and resolvedAt", () => { + expect(buildBillingLimitResolveDedupeKey("org_123", "2026-06-16T12:00:00.000Z")).toBe( + "billing-limit-resolve:org_123:2026-06-16T12:00:00.000Z", + ); + }); + + it("classifies converge failures for ops triage", () => { + expect(classifyPendingBillingLimitResolveConvergeFailure("new_only")).toBe("cancel-failing"); + expect(classifyPendingBillingLimitResolveConvergeFailure("queue")).toBe("converge-failing"); + }); +}); + +describe("runPendingBillingLimitResolves", () => { + const pending: PendingBillingLimitResolve = { + organizationId: "org_123", + resumeMode: "new_only", + resolvedAt: "2026-06-17T12:00:00.000Z", + }; + + it("keeps org pending and skips ack when converge throws (cancel-failing path)", async () => { + const completeCalls: string[] = []; + + const stillPending = await runPendingBillingLimitResolves([pending], { + converge: async () => { + throw new Error("bulk cancel failed"); + }, + complete: async (organizationId) => { + completeCalls.push(organizationId); + return { completed: true }; + }, + }); + + expect(stillPending).toEqual(new Set(["org_123"])); + expect(completeCalls).toEqual([]); + }); + + it("keeps org pending when converge succeeds but ack throws (ack-only path)", async () => { + let convergeCalls = 0; + let completeCalls = 0; + + const stillPending = await runPendingBillingLimitResolves( + [{ ...pending, resumeMode: "queue" }], + { + converge: async () => { + convergeCalls += 1; + }, + complete: async () => { + completeCalls += 1; + throw new Error("resolve-complete unavailable"); + }, + }, + ); + + expect(stillPending).toEqual(new Set(["org_123"])); + expect(convergeCalls).toBe(1); + expect(completeCalls).toBe(1); + }); + + it("keeps org pending when ack returns completed: false", async () => { + const stillPending = await runPendingBillingLimitResolves([pending], { + converge: async () => undefined, + complete: async () => ({ completed: false }), + }); + + expect(stillPending).toEqual(new Set(["org_123"])); + }); + + it("clears org from pending set when converge and ack both succeed", async () => { + const stillPending = await runPendingBillingLimitResolves([pending], { + converge: async () => undefined, + complete: async () => ({ completed: true }), + }); + + expect(stillPending).toEqual(new Set()); + }); + + it("retries ack on a later tick after a transient ack failure", async () => { + let ackAttempts = 0; + + const deps = { + converge: async () => undefined, + complete: async () => { + ackAttempts += 1; + if (ackAttempts === 1) { + throw new Error("resolve-complete unavailable"); + } + return { completed: true }; + }, + }; + + expect(await runPendingBillingLimitResolves([pending], deps)).toEqual(new Set(["org_123"])); + expect(await runPendingBillingLimitResolves([pending], deps)).toEqual(new Set()); + expect(ackAttempts).toBe(2); + }); +}); diff --git a/apps/webapp/test/billingLimitEnvCreatePause.test.ts b/apps/webapp/test/billingLimitEnvCreatePause.test.ts new file mode 100644 index 0000000000..0156959e4c --- /dev/null +++ b/apps/webapp/test/billingLimitEnvCreatePause.test.ts @@ -0,0 +1,78 @@ +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import { describe, expect, it } from "vitest"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { getInitialEnvPauseStateForBillingLimit } from "~/v3/services/billingLimit/getInitialEnvPauseStateForBillingLimit.server"; + +function configuredLimit(status: "grace" | "rejected" | "ok"): BillingLimitResult { + const hitAt = "2026-06-16T12:00:00.000Z"; + const graceEndsAt = "2026-06-17T12:00:00.000Z"; + + return { + isConfigured: true, + mode: "custom", + amountCents: 50_000, + cancelInProgressRuns: false, + effectiveAmountCents: 50_000, + gracePeriodMs: 86_400_000, + limitState: status === "ok" ? { status: "ok" } : { status, hitAt, graceEndsAt }, + }; +} + +describe("getInitialEnvPauseStateForBillingLimit", () => { + it("pauses billable environments when org is in grace", async () => { + const result = await getInitialEnvPauseStateForBillingLimit("org_123", "PRODUCTION", { + getBillingLimit: async () => configuredLimit("grace"), + }); + + expect(result).toEqual({ + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }); + }); + + it("pauses billable environments when org is rejected", async () => { + const result = await getInitialEnvPauseStateForBillingLimit("org_123", "STAGING", { + getBillingLimit: async () => configuredLimit("rejected"), + }); + + expect(result).toEqual({ + paused: true, + pauseSource: EnvironmentPauseSource.BILLING_LIMIT, + }); + }); + + it("does not pause development environments", async () => { + const result = await getInitialEnvPauseStateForBillingLimit("org_123", "DEVELOPMENT", { + getBillingLimit: async () => configuredLimit("rejected"), + }); + + expect(result).toEqual({ + paused: false, + pauseSource: null, + }); + }); + + it("does not pause when billing limit lookup fails", async () => { + const result = await getInitialEnvPauseStateForBillingLimit("org_123", "PRODUCTION", { + getBillingLimit: async () => { + throw new Error("billing platform unavailable"); + }, + }); + + expect(result).toEqual({ + paused: false, + pauseSource: null, + }); + }); + + it("does not pause when billing limit is ok", async () => { + const result = await getInitialEnvPauseStateForBillingLimit("org_123", "PRODUCTION", { + getBillingLimit: async () => configuredLimit("ok"), + }); + + expect(result).toEqual({ + paused: false, + pauseSource: null, + }); + }); +}); diff --git a/apps/webapp/test/billingLimitHit.test.ts b/apps/webapp/test/billingLimitHit.test.ts new file mode 100644 index 0000000000..ff986a4096 --- /dev/null +++ b/apps/webapp/test/billingLimitHit.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { BillingLimitHitWebhookBodySchema } from "~/services/billingLimit.schemas"; +import { + type BillingLimitHitDeps, + processBillingLimitHit, +} from "~/v3/services/billingLimit/billingLimitHit.server"; + +describe("billingLimitHit", () => { + it("busts caches, seeds reconcile, and enqueues grace converge", async () => { + const calls: string[] = []; + + const deps: BillingLimitHitDeps = { + bustCaches: (organizationId) => { + calls.push(`bust:${organizationId}`); + }, + seedReconcileQueue: async (organizationId) => { + calls.push(`seed:${organizationId}`); + }, + enqueueConverge: async (organizationId, targetState) => { + calls.push(`converge:${organizationId}:${targetState}`); + }, + enqueueCancelInProgressRuns: async () => { + calls.push("cancel"); + }, + }; + + await processBillingLimitHit( + { + organizationId: "org_123", + hitAt: "2026-06-16T12:00:00.000Z", + cancelInProgressRuns: false, + }, + deps, + ); + + expect(calls).toEqual(["bust:org_123", "seed:org_123", "converge:org_123:grace"]); + }); + + it("enqueues in-progress cancel when cancelInProgressRuns is true", async () => { + const cancelCalls: Array<{ organizationId: string; hitAt: string }> = []; + + const deps: BillingLimitHitDeps = { + bustCaches: () => {}, + seedReconcileQueue: async () => {}, + enqueueConverge: async () => {}, + enqueueCancelInProgressRuns: async (organizationId, hitAt) => { + cancelCalls.push({ organizationId, hitAt }); + }, + }; + + await processBillingLimitHit( + { + organizationId: "org_123", + hitAt: "2026-06-16T12:00:00.000Z", + cancelInProgressRuns: true, + }, + deps, + ); + + expect(cancelCalls).toEqual([{ organizationId: "org_123", hitAt: "2026-06-16T12:00:00.000Z" }]); + }); +}); + +describe("BillingLimitHitWebhookBodySchema", () => { + it("parses the hit webhook body", () => { + expect( + BillingLimitHitWebhookBodySchema.parse({ + hitAt: "2026-06-16T12:00:00.000Z", + cancelInProgressRuns: true, + limitState: "grace", + }), + ).toEqual({ + hitAt: "2026-06-16T12:00:00.000Z", + cancelInProgressRuns: true, + limitState: "grace", + }); + }); +}); diff --git a/apps/webapp/test/billingLimitPauseEnvironment.test.ts b/apps/webapp/test/billingLimitPauseEnvironment.test.ts new file mode 100644 index 0000000000..8536a451b9 --- /dev/null +++ b/apps/webapp/test/billingLimitPauseEnvironment.test.ts @@ -0,0 +1,26 @@ +import { EnvironmentPauseSource } from "@trigger.dev/database"; +import { describe, expect, it } from "vitest"; +import { getManualPauseEnvironmentResult } from "~/v3/services/billingLimit/manualPauseEnvironmentGuard.server"; + +describe("manualPauseEnvironmentGuard", () => { + it("blocks resume and no-ops pause for billing-paused environments", () => { + expect( + getManualPauseEnvironmentResult("resumed", EnvironmentPauseSource.BILLING_LIMIT), + ).toEqual({ + proceed: false, + success: false, + error: expect.stringContaining("billing limit"), + }); + + expect(getManualPauseEnvironmentResult("paused", EnvironmentPauseSource.BILLING_LIMIT)).toEqual( + { + proceed: false, + success: true, + state: "paused", + }, + ); + + expect(getManualPauseEnvironmentResult("resumed", null)).toEqual({ proceed: true }); + expect(getManualPauseEnvironmentResult("paused", null)).toEqual({ proceed: true }); + }); +}); diff --git a/apps/webapp/test/billingLimitQueuedRuns.test.ts b/apps/webapp/test/billingLimitQueuedRuns.test.ts new file mode 100644 index 0000000000..18aade2ede --- /dev/null +++ b/apps/webapp/test/billingLimitQueuedRuns.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, vi } from "vitest"; +import { setTimeout } from "node:timers/promises"; +import { replicationContainerTest } from "@internal/testcontainers"; +import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; +import { countQueuedRunsForBillableEnvironment } from "~/v3/services/billingLimit/billingLimitQueuedRuns.server"; +import { setupClickhouseReplication } from "./utils/replicationUtils"; + +vi.setConfig({ testTimeout: 60_000 }); + +describe("billingLimitQueuedRuns", () => { + replicationContainerTest( + "counts queued runs via RunsRepository.countRuns (same source as bulk cancel)", + async ({ clickhouseContainer, redisOptions, postgresContainer, prisma }) => { + const { clickhouse } = await setupClickhouseReplication({ + prisma, + databaseUrl: postgresContainer.getConnectionUri(), + clickhouseUrl: clickhouseContainer.getConnectionUrl(), + redisOptions, + }); + + const organization = await prisma.organization.create({ + data: { title: "billing-limit-queued", slug: "billing-limit-queued" }, + }); + + const project = await prisma.project.create({ + data: { + name: "billing-limit-queued", + slug: "billing-limit-queued", + organizationId: organization.id, + externalRef: "billing-limit-queued", + }, + }); + + const productionEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: "prod", + pkApiKey: "prod", + shortcode: "prod", + }, + }); + + const developmentEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: project.id, + organizationId: organization.id, + apiKey: "dev", + pkApiKey: "dev", + shortcode: "dev", + }, + }); + + await prisma.taskRun.create({ + data: { + friendlyId: "run_queued_prod", + taskIdentifier: "queued-task", + status: "PENDING", + payload: JSON.stringify({}), + traceId: "trace", + spanId: "span", + queue: "main", + runtimeEnvironmentId: productionEnv.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "PRODUCTION", + engine: "V2", + }, + }); + + await prisma.taskRun.create({ + data: { + friendlyId: "run_queued_dev", + taskIdentifier: "queued-task", + status: "PENDING", + payload: JSON.stringify({}), + traceId: "trace", + spanId: "span", + queue: "main", + runtimeEnvironmentId: developmentEnv.id, + projectId: project.id, + organizationId: organization.id, + environmentType: "DEVELOPMENT", + engine: "V2", + }, + }); + + await setTimeout(1000); + + const runsRepository = new RunsRepository({ prisma, clickhouse }); + + const productionCount = await countQueuedRunsForBillableEnvironment( + runsRepository, + organization.id, + { id: productionEnv.id, projectId: project.id }, + ); + + const developmentCount = await countQueuedRunsForBillableEnvironment( + runsRepository, + organization.id, + { id: developmentEnv.id, projectId: project.id }, + ); + + expect(productionCount).toBe(1); + expect(developmentCount).toBe(1); + }, + ); +}); diff --git a/apps/webapp/test/billingLimitReconcileTick.test.ts b/apps/webapp/test/billingLimitReconcileTick.test.ts new file mode 100644 index 0000000000..b94be90462 --- /dev/null +++ b/apps/webapp/test/billingLimitReconcileTick.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from "vitest"; +import { runBillingLimitReconcileTick } from "~/v3/services/billingLimit/runBillingLimitReconcileTick.server"; +import type { PendingBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitPendingResolve.types"; + +describe("runBillingLimitReconcileTick", () => { + const pending: PendingBillingLimitResolve = { + organizationId: "org_pending", + resumeMode: "queue", + resolvedAt: "2026-06-17T12:00:00.000Z", + }; + + it("runs pending resolves before collecting orgs and excludes still-pending orgs", async () => { + const order: string[] = []; + + await runBillingLimitReconcileTick({ + getPendingResolves: async () => ({ orgs: [pending] }), + runPendingResolves: async (pendingResolves) => { + order.push(`pending:${pendingResolves.map((row) => row.organizationId).join(",")}`); + return new Set(["org_pending"]); + }, + collectOrgs: async (options) => { + order.push(`collect:${[...(options?.excludeOrgIds ?? [])].join(",")}`); + return { + targets: [{ organizationId: "org_active", targetState: "grace" }], + queuedOrgIds: ["org_active"], + }; + }, + reconcileTarget: async (target) => { + order.push(`reconcile:${target.organizationId}:${target.targetState}`); + }, + clearProcessedQueue: async (queuedOrgIds, processedOrgIds) => { + order.push(`clear:${queuedOrgIds.join(",")}:${processedOrgIds.join(",")}`); + }, + bustCaches: () => {}, + enqueueConverge: async () => undefined, + }); + + expect(order).toEqual([ + "pending:org_pending", + "collect:org_pending", + "reconcile:org_active:grace", + "clear:org_active:org_active", + ]); + }); + + it("reconciles collected targets when no pending resolves remain", async () => { + const reconciled: Array<{ organizationId: string; targetState: string }> = []; + + await runBillingLimitReconcileTick({ + getPendingResolves: async () => ({ orgs: [] }), + runPendingResolves: async () => new Set(), + collectOrgs: async () => ({ + targets: [ + { organizationId: "org_grace", targetState: "grace" }, + { organizationId: "org_ok", targetState: "ok" }, + ], + queuedOrgIds: ["org_grace", "org_ok"], + }), + reconcileTarget: async (target, deps) => { + reconciled.push(target); + await deps.enqueueConverge(target.organizationId, target.targetState); + }, + clearProcessedQueue: async () => undefined, + bustCaches: () => {}, + enqueueConverge: async (organizationId, targetState) => { + reconciled.push({ organizationId, targetState: `enqueued:${targetState}` }); + }, + }); + + expect(reconciled).toEqual([ + { organizationId: "org_grace", targetState: "grace" }, + { organizationId: "org_grace", targetState: "enqueued:grace" }, + { organizationId: "org_ok", targetState: "ok" }, + { organizationId: "org_ok", targetState: "enqueued:ok" }, + ]); + }); + + it("continues reconciling other targets and only clears successfully processed orgs", async () => { + const reconciled: string[] = []; + let clearedProcessedOrgIds: string[] = []; + + await runBillingLimitReconcileTick({ + getPendingResolves: async () => ({ orgs: [] }), + runPendingResolves: async () => new Set(), + collectOrgs: async () => ({ + targets: [ + { organizationId: "org_fail", targetState: "grace" }, + { organizationId: "org_ok", targetState: "ok" }, + ], + queuedOrgIds: ["org_fail", "org_ok"], + }), + reconcileTarget: async (target) => { + if (target.organizationId === "org_fail") { + throw new Error("reconcile failed"); + } + reconciled.push(target.organizationId); + }, + clearProcessedQueue: async (_queuedOrgIds, processedOrgIds) => { + clearedProcessedOrgIds = processedOrgIds; + }, + bustCaches: () => {}, + enqueueConverge: async () => undefined, + }); + + expect(reconciled).toEqual(["org_ok"]); + expect(clearedProcessedOrgIds).toEqual(["org_ok"]); + }); +}); diff --git a/apps/webapp/test/billingLimitReconciliation.test.ts b/apps/webapp/test/billingLimitReconciliation.test.ts new file mode 100644 index 0000000000..e206243298 --- /dev/null +++ b/apps/webapp/test/billingLimitReconciliation.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import type { BillingLimitResult } from "~/services/billingLimit.schemas"; +import { + collectOrgIdsNeedingBillingLimitLookup, + resolveConvergeTargetFromBillingLimit, +} from "~/v3/services/billingLimit/billingLimitReconciliation.server"; + +const graceLimit: BillingLimitResult = { + isConfigured: true, + mode: "custom", + amountCents: 10_000, + cancelInProgressRuns: false, + limitState: { + status: "grace", + hitAt: "2026-01-01T00:00:00.000Z", + graceEndsAt: "2026-01-02T00:00:00.000Z", + }, + effectiveAmountCents: 10_000, + gracePeriodMs: 86_400_000, +}; + +describe("billingLimitReconciliation", () => { + it("maps grace and rejected limits to converge targets", () => { + expect(resolveConvergeTargetFromBillingLimit(graceLimit)).toBe("grace"); + expect( + resolveConvergeTargetFromBillingLimit({ + ...graceLimit, + limitState: { + status: "rejected", + hitAt: "2026-01-01T00:00:00.000Z", + graceEndsAt: "2026-01-02T00:00:00.000Z", + }, + }), + ).toBe("rejected"); + expect( + resolveConvergeTargetFromBillingLimit({ + ...graceLimit, + limitState: { status: "ok" }, + }), + ).toBe("ok"); + expect(resolveConvergeTargetFromBillingLimit(undefined)).toBe("ok"); + expect( + resolveConvergeTargetFromBillingLimit({ isConfigured: false, gracePeriodMs: 86_400_000 }), + ).toBe("ok"); + }); + + it("dedupes stale and queued org ids and skips excluded or already-covered orgs", () => { + expect( + collectOrgIdsNeedingBillingLimitLookup({ + staleOrgIds: ["org_a", "org_b", "org_c"], + queuedOrgIds: ["org_b", "org_d", "org_a"], + excludeOrgIds: new Set(["org_c"]), + coveredOrgIds: new Set(["org_d"]), + }), + ).toEqual(["org_a", "org_b"]); + }); +}); diff --git a/apps/webapp/test/billingLimitResolve.test.ts b/apps/webapp/test/billingLimitResolve.test.ts new file mode 100644 index 0000000000..3cc6148038 --- /dev/null +++ b/apps/webapp/test/billingLimitResolve.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { processBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitResolve.server"; +import type { PendingBillingLimitResolve } from "~/v3/services/billingLimit/billingLimitPendingResolve.types"; + +describe("processBillingLimitResolve", () => { + const pending: PendingBillingLimitResolve = { + organizationId: "org_123", + resumeMode: "queue", + resolvedAt: "2026-06-17T12:00:00.000Z", + }; + + it("busts caches and enqueues resolve work", async () => { + const busted: string[] = []; + const enqueued: PendingBillingLimitResolve[] = []; + + await processBillingLimitResolve(pending, { + bustCaches: (organizationId) => { + busted.push(organizationId); + }, + enqueueResolve: async (payload) => { + enqueued.push(payload); + }, + }); + + expect(busted).toEqual(["org_123"]); + expect(enqueued).toEqual([pending]); + }); +}); diff --git a/apps/webapp/test/billingLimitTriggerEntitlement.test.ts b/apps/webapp/test/billingLimitTriggerEntitlement.test.ts new file mode 100644 index 0000000000..591318eae1 --- /dev/null +++ b/apps/webapp/test/billingLimitTriggerEntitlement.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { validateProductionEntitlement } from "~/runEngine/validators/validateProductionEntitlement.server"; + +const productionEnv = { + type: "PRODUCTION" as const, + organizationId: "org_123", +}; + +const developmentEnv = { + type: "DEVELOPMENT" as const, + organizationId: "org_123", +}; + +describe("validateProductionEntitlement", () => { + it("allows development environments without checking entitlement", async () => { + const result = await validateProductionEntitlement( + { environment: developmentEnv as never }, + async () => ({ hasAccess: false, reason: "billing_limit" }), + ); + + expect(result).toEqual({ ok: true }); + }); + + it("rejects production triggers when entitlement has billing_limit denial", async () => { + const result = await validateProductionEntitlement( + { environment: productionEnv as never }, + async () => ({ + hasAccess: false, + reason: "billing_limit", + plan: { type: "paid", code: "pro", isPaying: true }, + }), + ); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.name).toBe("OutOfEntitlementError"); + } + }); + + it("allows production triggers when entitlement grants access", async () => { + const plan = { type: "paid" as const, code: "pro", isPaying: true }; + + const result = await validateProductionEntitlement( + { environment: productionEnv as never }, + async () => ({ + hasAccess: true, + plan, + }), + ); + + expect(result).toEqual({ ok: true, plan }); + }); +}); diff --git a/apps/webapp/test/billingLimitsRoute.test.ts b/apps/webapp/test/billingLimitsRoute.test.ts new file mode 100644 index 0000000000..a61c9f0005 --- /dev/null +++ b/apps/webapp/test/billingLimitsRoute.test.ts @@ -0,0 +1,349 @@ +import { describe, expect, it } from "vitest"; +import { parse } from "@conform-to/zod"; +import { billingAlertsSchema } from "~/components/billing/BillingAlertsSection"; +import { + billingLimitFormSchema, + getBillingLimitFormLastSubmission, + isBillingLimitFormDirty, +} from "~/components/billing/BillingLimitConfigSection"; +import { billingLimitRecoveryFormSchema } from "~/components/billing/BillingLimitRecoveryPanel"; +import { isBillingLimitSettingsFormSubmission } from "~/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRevalidation"; +import { getSuggestedRecoveryLimitDollars } from "~/components/billing/billingLimitFormat"; +import { + getAlertsResetRequested, + getEffectiveLimitCentsAfterLimitSave, + getResolveSubmitted, + getSubmittedResumeMode, + isEnforcementActive, +} from "~/routes/_app.orgs.$organizationSlug.settings.billing-limits/billingLimitsRoute.server"; +import { loader as billingAlertsRedirectLoader } from "~/routes/_app.orgs.$organizationSlug.settings.billing-alerts/route"; + +function billingLimitsRequest(search = ""): Request { + return new Request(`http://localhost:3030/orgs/acme/settings/billing-limits${search}`); +} + +describe("billingLimitsRoute.server", () => { + describe("isEnforcementActive", () => { + it("returns true in grace", () => { + expect( + isEnforcementActive({ + isConfigured: true, + mode: "plan", + cancelInProgressRuns: false, + limitState: { status: "grace", hitAt: "t", graceEndsAt: "t" }, + effectiveAmountCents: 5000, + gracePeriodMs: 86_400_000, + }), + ).toBe(true); + }); + + it("returns true when rejected", () => { + expect( + isEnforcementActive({ + isConfigured: true, + mode: "custom", + amountCents: 2500, + cancelInProgressRuns: false, + limitState: { status: "rejected", hitAt: "t", graceEndsAt: "t" }, + effectiveAmountCents: 2500, + gracePeriodMs: 86_400_000, + }), + ).toBe(true); + }); + + it("returns false when unconfigured", () => { + expect( + isEnforcementActive({ + isConfigured: false, + gracePeriodMs: 86_400_000, + }), + ).toBe(false); + }); + + it("returns false when configured and ok", () => { + expect( + isEnforcementActive({ + isConfigured: true, + mode: "none", + cancelInProgressRuns: false, + limitState: { status: "ok" }, + effectiveAmountCents: null, + gracePeriodMs: 86_400_000, + }), + ).toBe(false); + }); + }); + + describe("getAlertsResetRequested", () => { + it("returns true when alertsReset=1 is present", () => { + expect(getAlertsResetRequested(billingLimitsRequest("?alertsReset=1"))).toBe(true); + }); + + it("returns false when the param is absent", () => { + expect(getAlertsResetRequested(billingLimitsRequest())).toBe(false); + }); + + it("returns false for other param values", () => { + expect(getAlertsResetRequested(billingLimitsRequest("?alertsReset=true"))).toBe(false); + }); + }); + + describe("getResolveSubmitted", () => { + it("returns true when resolved=1 is present", () => { + expect(getResolveSubmitted(billingLimitsRequest("?resolved=1"))).toBe(true); + }); + + it("returns false when the param is absent", () => { + expect(getResolveSubmitted(billingLimitsRequest())).toBe(false); + }); + }); + + describe("getSubmittedResumeMode", () => { + it("parses resumeMode from the query string", () => { + expect(getSubmittedResumeMode(billingLimitsRequest("?resumeMode=new_only"))).toBe("new_only"); + }); + + it("returns null for invalid values", () => { + expect(getSubmittedResumeMode(billingLimitsRequest("?resumeMode=invalid"))).toBeNull(); + }); + }); + + describe("getSuggestedRecoveryLimitDollars", () => { + it("uses max(limit + $50, limit × 1.25, spend × 1.25) rounded up to $50", () => { + expect(getSuggestedRecoveryLimitDollars(5_000, 4_500)).toBe(100); + expect(getSuggestedRecoveryLimitDollars(50_000, 48_000)).toBe(650); + expect(getSuggestedRecoveryLimitDollars(50_000, 60_000)).toBe(750); + expect(getSuggestedRecoveryLimitDollars(1_000_000, 950_000)).toBe(12_500); + }); + + it("falls back to spend × 1.25 when there is no effective limit", () => { + expect(getSuggestedRecoveryLimitDollars(null, 4_500)).toBe(100); + }); + }); + + describe("isBillingLimitSettingsFormSubmission", () => { + it("returns true for billing-limit POST", () => { + const formData = new FormData(); + formData.set("intent", "billing-limit"); + expect(isBillingLimitSettingsFormSubmission("post", formData)).toBe(true); + }); + + it("returns true for billing-alerts POST", () => { + const formData = new FormData(); + formData.set("intent", "billing-alerts"); + expect(isBillingLimitSettingsFormSubmission("POST", formData)).toBe(true); + }); + + it("returns true for billing-limit-resolve POST", () => { + const formData = new FormData(); + formData.set("intent", "billing-limit-resolve"); + expect(isBillingLimitSettingsFormSubmission("post", formData)).toBe(true); + }); + + it("returns false for unrelated POST", () => { + const formData = new FormData(); + formData.set("intent", "other"); + expect(isBillingLimitSettingsFormSubmission("post", formData)).toBe(false); + }); + + it("returns false without form data", () => { + expect(isBillingLimitSettingsFormSubmission("post", undefined)).toBe(false); + }); + }); + + describe("getEffectiveLimitCentsAfterLimitSave", () => { + it("uses custom amount in cents for custom mode", () => { + expect(getEffectiveLimitCentsAfterLimitSave("custom", 5000, 42.5)).toBe(4250); + }); + + it("uses plan limit cents for plan mode", () => { + expect(getEffectiveLimitCentsAfterLimitSave("plan", 5000)).toBe(5000); + }); + + it("uses plan limit cents for none mode", () => { + expect(getEffectiveLimitCentsAfterLimitSave("none", 5000)).toBe(5000); + }); + }); +}); + +describe("billing-alerts redirect route", () => { + it("redirects to billing-limits", async () => { + const response = await billingAlertsRedirectLoader({ + params: { organizationSlug: "acme" }, + request: new Request("http://localhost:3030/orgs/acme/settings/billing-alerts"), + context: {}, + }); + + expect(response.status).toBe(302); + expect(response.headers.get("Location")).toBe("/orgs/acme/settings/billing-limits"); + }); +}); + +describe("billing-limits form validation", () => { + it("rejects duplicate alert thresholds", () => { + const formData = new FormData(); + formData.set("intent", "billing-alerts"); + formData.append("emails", "a@example.com"); + formData.append("alertLevels", "75"); + formData.append("alertLevels", "75"); + + const submission = parse(formData, { schema: billingAlertsSchema }); + expect(submission.error?.alertLevels).toBeTruthy(); + }); + + it("rejects non-numeric alert thresholds", () => { + const formData = new FormData(); + formData.set("intent", "billing-alerts"); + formData.append("emails", "a@example.com"); + formData.append("alertLevels", "75"); + formData.append("alertLevels", "not-a-number"); + + const submission = parse(formData, { schema: billingAlertsSchema }); + expect(submission.error?.["alertLevels[1]"]).toBeTruthy(); + }); + + it("accepts a valid billing limit custom submission", () => { + const formData = new FormData(); + formData.set("intent", "billing-limit"); + formData.set("mode", "custom"); + formData.set("amount", "100"); + formData.set("cancelInProgressRuns", "on"); + + const submission = parse(formData, { schema: billingLimitFormSchema }); + expect(submission.value).toEqual({ + mode: "custom", + amount: 100, + cancelInProgressRuns: true, + }); + }); + + it("parses none mode with cancelInProgressRuns from the form", () => { + const formData = new FormData(); + formData.set("mode", "none"); + formData.set("cancelInProgressRuns", "on"); + + const submission = parse(formData, { schema: billingLimitFormSchema }); + expect(submission.value?.mode).toBe("none"); + expect(submission.value?.cancelInProgressRuns).toBe(true); + }); + + it("accepts a valid billing limit resolve submission", () => { + const formData = new FormData(); + formData.set("intent", "billing-limit-resolve"); + formData.set("action", "increase"); + formData.set("newAmount", "1500"); + formData.set("resumeMode", "queue"); + + const submission = parse(formData, { schema: billingLimitRecoveryFormSchema }); + expect(submission.value).toEqual({ + action: "increase", + newAmount: 1500, + resumeMode: "queue", + }); + }); + + it("accepts remove resolve with new_only resume mode", () => { + const formData = new FormData(); + formData.set("action", "remove"); + formData.set("resumeMode", "new_only"); + + const submission = parse(formData, { schema: billingLimitRecoveryFormSchema }); + expect(submission.value).toEqual({ + action: "remove", + resumeMode: "new_only", + }); + }); +}); + +describe("isBillingLimitFormDirty", () => { + const unconfiguredLimit = { isConfigured: false as const, gracePeriodMs: 86_400_000 }; + const configuredPlanLimit = { + isConfigured: true as const, + mode: "plan" as const, + cancelInProgressRuns: false, + limitState: { status: "ok" as const }, + effectiveAmountCents: 5000, + gracePeriodMs: 86_400_000, + }; + + it("is dirty when billing limit has never been saved", () => { + expect( + isBillingLimitFormDirty({ + billingLimit: unconfiguredLimit, + mode: "none", + customAmount: "", + cancelInProgressRuns: false, + }), + ).toBe(true); + }); + + it("is clean when configured values match saved state", () => { + expect( + isBillingLimitFormDirty({ + billingLimit: configuredPlanLimit, + mode: "plan", + customAmount: "", + cancelInProgressRuns: false, + }), + ).toBe(false); + }); + + it("is dirty when configured mode changes", () => { + expect( + isBillingLimitFormDirty({ + billingLimit: configuredPlanLimit, + mode: "none", + customAmount: "", + cancelInProgressRuns: false, + }), + ).toBe(true); + }); +}); + +describe("getBillingLimitFormLastSubmission", () => { + it("drops amount errors when the selected mode is not custom", () => { + const submission = parse( + (() => { + const formData = new FormData(); + formData.set("mode", "custom"); + formData.set("amount", "0"); + return formData; + })(), + { schema: billingLimitFormSchema }, + ); + + expect( + getBillingLimitFormLastSubmission(submission, "plan", true)?.error?.amount, + ).toBeUndefined(); + }); + + it("keeps amount errors while custom mode is selected", () => { + const submission = parse( + (() => { + const formData = new FormData(); + formData.set("mode", "custom"); + formData.set("amount", "0"); + return formData; + })(), + { schema: billingLimitFormSchema }, + ); + + expect( + getBillingLimitFormLastSubmission(submission, "custom", true)?.error?.amount, + ).toBeTruthy(); + }); + + it("returns undefined when the form is clean", () => { + const submission = parse( + (() => { + const formData = new FormData(); + formData.set("mode", "custom"); + formData.set("amount", "0"); + return formData; + })(), + { schema: billingLimitFormSchema }, + ); + + expect(getBillingLimitFormLastSubmission(submission, "custom", false)).toBeUndefined(); + }); +}); diff --git a/apps/webapp/test/orgBanner.test.ts b/apps/webapp/test/orgBanner.test.ts new file mode 100644 index 0000000000..10c7c1e928 --- /dev/null +++ b/apps/webapp/test/orgBanner.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { OrgBannerKind, selectOrgBanner } from "~/components/billing/selectOrgBanner"; + +describe("selectOrgBanner", () => { + it("prioritizes limit-rejected over grace and no-limit", () => { + expect( + selectOrgBanner({ + billingLimit: { + isConfigured: true, + mode: "plan", + cancelInProgressRuns: false, + limitState: { status: "rejected", hitAt: "t", graceEndsAt: "t" }, + effectiveAmountCents: 1000, + gracePeriodMs: 86_400_000, + }, + hasExceededFreeTier: true, + showEnvironmentWarning: true, + }), + ).toBe(OrgBannerKind.LimitRejected); + }); + + it("prioritizes upgrade over no-limit when free tier is exceeded", () => { + expect( + selectOrgBanner({ + billingLimit: { isConfigured: false, gracePeriodMs: 86_400_000 }, + hasExceededFreeTier: true, + showSelfServe: true, + }), + ).toBe(OrgBannerKind.Upgrade); + }); + + it("shows no-limit when unconfigured and self-serve", () => { + expect( + selectOrgBanner({ + billingLimit: { isConfigured: false, gracePeriodMs: 86_400_000 }, + hasExceededFreeTier: false, + showSelfServe: true, + }), + ).toBe(OrgBannerKind.NoLimitConfigured); + }); + + it("hides no-limit when unconfigured but not self-serve", () => { + expect( + selectOrgBanner({ + billingLimit: { isConfigured: false, gracePeriodMs: 86_400_000 }, + hasExceededFreeTier: true, + showSelfServe: false, + }), + ).toBe(OrgBannerKind.Upgrade); + }); + + it("hides no-limit prompt when configured with mode none", () => { + expect( + selectOrgBanner({ + billingLimit: { + isConfigured: true, + mode: "none", + cancelInProgressRuns: false, + limitState: { status: "ok" }, + effectiveAmountCents: null, + gracePeriodMs: 86_400_000, + }, + }), + ).toBe(OrgBannerKind.None); + }); +}); diff --git a/docs/how-to-reduce-your-spend.mdx b/docs/how-to-reduce-your-spend.mdx index c070745d43..13ec16821b 100644 --- a/docs/how-to-reduce-your-spend.mdx +++ b/docs/how-to-reduce-your-spend.mdx @@ -16,21 +16,28 @@ Monitor your usage dashboard to understand your spending patterns. You can see: You can view your usage page by clicking the "Organization" menu in the top left of the dashboard and then clicking "Usage". -## Create billing alerts +## Set billing limits and alerts -Configure billing alerts in your dashboard to get notified when you approach spending thresholds. This helps you: +Configure billing limits and alerts in your dashboard to protect against unexpected usage and get notified when you approach spending thresholds. This helps you: +- Set a monthly compute spend limit for your organization - Catch unexpected cost increases early - Identify runaway tasks before they become expensive -The billing alerts page includes two types of alerts: +The **Billing limits** settings page has two sections: -- **Standard alerts**: Get notified at 75%, 90%, 100%, 200%, and 500% of your monthly budget -- **Spike alerts**: Catch runaway usage from bugs or errors with alerts at 10x (1000%), 20x (2000%), 50x (5000%), and 100x (10000%) of your monthly budget. We recommend keeping these enabled as a safety net. +- **Billing limit**: Choose your plan limit, a custom amount, or no limit. When a limit is reached, billable environments (`production`, `staging`, and `preview`) enter a **grace period** — queues pause and new runs queue without starting. After grace expires, new triggers are rejected until you increase or remove the limit. +- **Billing alerts**: Add email alerts at specific spend thresholds (% of your limit when a limit is set, or dollar amounts when no limit is configured). Alerts notify you only; they do **not** pause environments or reject triggers. -![Billing alerts](./images/billing-alerts-ui.png) +**Limits vs alerts:** A billing limit enforces spend (grace → reject). Billing alerts are optional notifications at thresholds you choose. -You can view your billing alerts page by clicking the "Organization" menu in the top left of the dashboard and then clicking "Settings". +**Soft limits:** Billing limits are not instantaneous hard caps. Usage is evaluated on a short delay, so spend can briefly exceed your limit before enforcement applies. Queued runs during grace incur no compute cost until they start. See our [terms](https://trigger.dev/terms) for refund policy details. + +On the **Usage** page, when you have a custom billing limit (or a plan limit that differs from included usage), a **Billing limit** marker appears on the usage bar alongside your current spend and plan included usage. + +![Billing limits and alerts settings](./images/billing-alerts-ui.png) + +You can open the page from the **Organization** menu in the top left of the dashboard, then **Settings** → **Billing limits**. ## Reduce your machine sizes diff --git a/internal-packages/database/prisma/migrations/20260614120000_add_environment_pause_source_billing_limit/migration.sql b/internal-packages/database/prisma/migrations/20260614120000_add_environment_pause_source_billing_limit/migration.sql new file mode 100644 index 0000000000..d34b00e568 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260614120000_add_environment_pause_source_billing_limit/migration.sql @@ -0,0 +1,5 @@ +-- CreateEnum +CREATE TYPE "EnvironmentPauseSource" AS ENUM ('BILLING_LIMIT'); + +-- AlterTable +ALTER TABLE "RuntimeEnvironment" ADD COLUMN "pauseSource" "EnvironmentPauseSource"; diff --git a/internal-packages/database/prisma/migrations/20260626120000_bulk_action_group_dedupe_key_column/migration.sql b/internal-packages/database/prisma/migrations/20260626120000_bulk_action_group_dedupe_key_column/migration.sql new file mode 100644 index 0000000000..94181bc27b --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260626120000_bulk_action_group_dedupe_key_column/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "BulkActionGroup" ADD COLUMN IF NOT EXISTS "dedupeKey" TEXT; diff --git a/internal-packages/database/prisma/migrations/20260626120001_bulk_action_group_dedupe_key_index/migration.sql b/internal-packages/database/prisma/migrations/20260626120001_bulk_action_group_dedupe_key_index/migration.sql new file mode 100644 index 0000000000..189c4d0888 --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260626120001_bulk_action_group_dedupe_key_index/migration.sql @@ -0,0 +1,3 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "BulkActionGroup_environmentId_type_dedupeKey_idx" +ON "BulkActionGroup" ("environmentId", "type", "dedupeKey") +WHERE "dedupeKey" IS NOT NULL; diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index a953d6a367..73be7b952a 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -343,6 +343,7 @@ model RuntimeEnvironment { maximumConcurrencyLimit Int @default(5) concurrencyLimitBurstFactor Decimal @default("2.00") @db.Decimal(4, 2) paused Boolean @default(false) + pauseSource EnvironmentPauseSource? autoEnableInternalSources Boolean @default(true) @@ -422,6 +423,10 @@ enum RuntimeEnvironmentType { PREVIEW } +enum EnvironmentPauseSource { + BILLING_LIMIT +} + model Project { id String @id @default(cuid()) slug String @unique @@ -2492,6 +2497,8 @@ model BulkActionGroup { queryName String? /// The params that will be passed to the query params Json? + /// Billing-limit dedupe key (grace hitAt or resolve job key). Null for dashboard bulk actions. + dedupeKey String? /// The cursor that will be passed to the query (null for the first page) cursor Json? /// The number of runs that have been processed successfully @@ -2517,6 +2524,7 @@ model BulkActionGroup { updatedAt DateTime @updatedAt @@index([environmentId, createdAt(sort: Desc)]) + @@index([environmentId, type, dedupeKey]) } enum BulkActionType {