feat: add text-to-drawing

excalidraw-app/index.tsx

@@ -104,6 +104,7 @@ import { ShareableLinkDialog } from "../src/components/ShareableLinkDialog";
 import { openConfirmModal } from "../src/components/OverwriteConfirm/OverwriteConfirmState";
 import { OverwriteConfirmDialog } from "../src/components/OverwriteConfirm/OverwriteConfirm";
 import Trans from "../src/components/Trans";
+import { drawingIcon } from "../src/components/icons";
 
 polyfill();
 
@@ -776,12 +777,10 @@ const ExcalidrawWrapper = () => {
         </OverwriteConfirmDialog>
         <AppFooter />
         <TTDDialog
-          onTextSubmit={async (input) => {
+          onTextSubmit={async (input, type) => {
             try {
               const response = await fetch(
-                `${
-                  import.meta.env.VITE_APP_AI_BACKEND
-                }/v1/ai/text-to-diagram/generate`,
+                `${import.meta.env.VITE_APP_AI_BACKEND}/v1/ai/${type}/generate`,
                 {
                   method: "POST",
                   headers: {
@@ -833,6 +832,9 @@ const ExcalidrawWrapper = () => {
           }}
         />
         <TTDDialogTrigger />
+        <TTDDialogTrigger tab="text-to-drawing" icon={drawingIcon}>
+          {t("labels.textToDrawing")}
+        </TTDDialogTrigger>
         {isCollaborating && isOffline && (
           <div className="collab-offline-warning">
             {t("alerts.collabOfflineWarning")}

src/components/App.tsx

@@ -397,7 +397,6 @@ import { COLOR_PALETTE } from "../colors";
 import { ElementCanvasButton } from "./MagicButton";
 import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
 import { EditorLocalStorage } from "../data/EditorLocalStorage";
-import { TextToExcalidraw } from "./TextToExcalidraw/TextToExcalidraw";
 
 const AppContext = React.createContext<AppClassProperties>(null!);
 const AppPropsContext = React.createContext<AppProps>(null!);

src/components/TTDDialog/MermaidToExcalidraw.tsx

@@ -111,7 +111,7 @@ const MermaidToExcalidraw = ({
             action: () => {
               insertToEditor({
                 app,
-                data,
+                data: data.current,
                 text,
                 shouldSaveMermaidDataToStorage: true,
               });

src/components/TTDDialog/TTDDialog.tsx

@@ -2,58 +2,20 @@ import { Dialog } from "../Dialog";
 import { useApp } from "../App";
 import MermaidToExcalidraw from "./MermaidToExcalidraw";
 import TTDDialogTabs from "./TTDDialogTabs";
-import { ChangeEventHandler, useEffect, useRef, useState } from "react";
+import { useEffect, useState } from "react";
 import { useUIAppState } from "../../context/ui-appState";
 import { withInternalFallback } from "../hoc/withInternalFallback";
 import { TTDDialogTabTriggers } from "./TTDDialogTabTriggers";
 import { TTDDialogTabTrigger } from "./TTDDialogTabTrigger";
 import { TTDDialogTab } from "./TTDDialogTab";
 import { t } from "../../i18n";
-import { TTDDialogInput } from "./TTDDialogInput";
-import { TTDDialogOutput } from "./TTDDialogOutput";
-import { TTDDialogPanel } from "./TTDDialogPanel";
-import { TTDDialogPanels } from "./TTDDialogPanels";
-import {
-  MermaidToExcalidrawLibProps,
-  convertMermaidToExcalidraw,
-  insertToEditor,
-  saveMermaidDataToStorage,
-} from "./common";
-import { NonDeletedExcalidrawElement } from "../../element/types";
-import { BinaryFiles } from "../../types";
-import { ArrowRightIcon } from "../icons";
+import { CommonDialogProps, MermaidToExcalidrawLibProps } from "./common";
 
 import "./TTDDialog.scss";
-import { isFiniteNumber } from "../../utils";
-import { atom, useAtom } from "jotai";
-import { trackEvent } from "../../analytics";
+import { TextToDiagram } from "./TextToDiagram";
+import { TextToDrawing } from "./TextToDrawing";
 
-const MIN_PROMPT_LENGTH = 3;
-const MAX_PROMPT_LENGTH = 1000;
-
-const rateLimitsAtom = atom<{
-  rateLimit: number;
-  rateLimitRemaining: number;
-} | null>(null);
-
-type OnTestSubmitRetValue = {
-  rateLimit?: number | null;
-  rateLimitRemaining?: number | null;
-} & (
-  | { generatedResponse: string | undefined; error?: null | undefined }
-  | {
-      error: Error;
-      generatedResponse?: null | undefined;
-    }
-);
-
-export const TTDDialog = (
-  props:
-    | {
-        onTextSubmit(value: string): Promise<OnTestSubmitRetValue>;
-      }
-    | { __fallback: true },
-) => {
+export const TTDDialog = (props: CommonDialogProps | { __fallback: true }) => {
   const appState = useUIAppState();
 
   if (appState.openDialog?.name !== "ttd") {
@@ -72,118 +34,10 @@ export const TTDDialogBase = withInternalFallback(
     tab,
     ...rest
   }: {
-    tab: "text-to-diagram" | "mermaid";
-  } & (
-    | {
-        onTextSubmit(value: string): Promise<OnTestSubmitRetValue>;
-      }
-    | { __fallback: true }
-  )) => {
+    tab: "text-to-diagram" | "mermaid" | "text-to-drawing";
+  } & (CommonDialogProps | { __fallback: true })) => {
     const app = useApp();
 
-    const someRandomDivRef = useRef<HTMLDivElement>(null);
-
-    const [text, setText] = useState("");
-
-    const prompt = text.trim();
-
-    const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (
-      event,
-    ) => {
-      setText(event.target.value);
-    };
-
-    const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
-    const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
-
-    const onGenerate = async () => {
-      if (
-        prompt.length > MAX_PROMPT_LENGTH ||
-        prompt.length < MIN_PROMPT_LENGTH ||
-        onTextSubmitInProgess ||
-        rateLimits?.rateLimitRemaining === 0 ||
-        // means this is not a text-to-diagram dialog (needed for TS only)
-        "__fallback" in rest
-      ) {
-        if (prompt.length < MIN_PROMPT_LENGTH) {
-          setError(
-            new Error(
-              `Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
-            ),
-          );
-        }
-        if (prompt.length > MAX_PROMPT_LENGTH) {
-          setError(
-            new Error(
-              `Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`,
-            ),
-          );
-        }
-
-        return;
-      }
-
-      try {
-        setOnTextSubmitInProgess(true);
-
-        trackEvent("ai", "generate", "ttd");
-
-        const { generatedResponse, error, rateLimit, rateLimitRemaining } =
-          await rest.onTextSubmit(prompt);
-
-        if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
-          setRateLimits({ rateLimit, rateLimitRemaining });
-        }
-
-        if (error) {
-          setError(error);
-          return;
-        }
-        if (!generatedResponse) {
-          setError(new Error("Generation failed"));
-          return;
-        }
-
-        try {
-          await convertMermaidToExcalidraw({
-            canvasRef: someRandomDivRef,
-            data,
-            mermaidToExcalidrawLib,
-            setError,
-            mermaidDefinition: generatedResponse,
-          });
-          trackEvent("ai", "mermaid parse success", "ttd");
-          saveMermaidDataToStorage(generatedResponse);
-        } catch (error: any) {
-          console.info(
-            `%cTTD mermaid render errror: ${error.message}`,
-            "color: red",
-          );
-          console.info(
-            `>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\nTTD mermaid definition render errror: ${error.message}`,
-            "color: yellow",
-          );
-          trackEvent("ai", "mermaid parse failed", "ttd");
-          setError(
-            new Error(
-              "Generated an invalid diagram :(. You may also try a different prompt.",
-            ),
-          );
-        }
-      } catch (error: any) {
-        let message: string | undefined = error.message;
-        if (!message || message === "Failed to fetch") {
-          message = "Request failed";
-        }
-        setError(new Error(message));
-      } finally {
-        setOnTextSubmitInProgess(false);
-      }
-    };
-
-    const refOnGenerate = useRef(onGenerate);
-    refOnGenerate.current = onGenerate;
-
     const [mermaidToExcalidrawLib, setMermaidToExcalidrawLib] =
       useState<MermaidToExcalidrawLibProps>({
         loaded: false,
@@ -200,13 +54,6 @@ export const TTDDialogBase = withInternalFallback(
       fn();
     }, [mermaidToExcalidrawLib.api]);
 
-    const data = useRef<{
-      elements: readonly NonDeletedExcalidrawElement[];
-      files: BinaryFiles | null;
-    }>({ elements: [], files: null });
-
-    const [error, setError] = useState<Error | null>(null);
-
     return (
       <Dialog
         className="ttd-dialog"
@@ -243,6 +90,9 @@ export const TTDDialogBase = withInternalFallback(
                   </div>
                 </div>
               </TTDDialogTabTrigger>
+              <TTDDialogTabTrigger tab="text-to-drawing">
+                {t("labels.textToDrawing")}
+              </TTDDialogTabTrigger>
               <TTDDialogTabTrigger tab="mermaid">Mermaid</TTDDialogTabTrigger>
             </TTDDialogTabTriggers>
           )}
@@ -254,93 +104,15 @@ export const TTDDialogBase = withInternalFallback(
           </TTDDialogTab>
           {!("__fallback" in rest) && (
             <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
-              <div className="ttd-dialog-desc">
-                Currently we use Mermaid as a middle step, so you'll get best
-                results if you describe a diagram, workflow, flow chart, and
-                similar.
-              </div>
-              <TTDDialogPanels>
-                <TTDDialogPanel
-                  label={t("labels.prompt")}
-                  panelAction={{
-                    action: onGenerate,
-                    label: "Generate",
-                    icon: ArrowRightIcon,
-                  }}
-                  onTextSubmitInProgess={onTextSubmitInProgess}
-                  panelActionDisabled={
-                    prompt.length > MAX_PROMPT_LENGTH ||
-                    rateLimits?.rateLimitRemaining === 0
-                  }
-                  renderTopRight={() => {
-                    if (!rateLimits) {
-                      return null;
-                    }
-
-                    return (
-                      <div
-                        className="ttd-dialog-rate-limit"
-                        style={{
-                          fontSize: 12,
-                          marginLeft: "auto",
-                          color:
-                            rateLimits.rateLimitRemaining === 0
-                              ? "var(--color-danger)"
-                              : undefined,
-                        }}
-                      >
-                        {rateLimits.rateLimitRemaining} requests left today
-                      </div>
-                    );
-                  }}
-                  renderBottomRight={() => {
-                    const ratio = prompt.length / MAX_PROMPT_LENGTH;
-                    if (ratio > 0.8) {
-                      return (
-                        <div
-                          style={{
-                            marginLeft: "auto",
-                            fontSize: 12,
-                            fontFamily: "monospace",
-                            color:
-                              ratio > 1 ? "var(--color-danger)" : undefined,
-                          }}
-                        >
-                          Length: {prompt.length}/{MAX_PROMPT_LENGTH}
-                        </div>
-                      );
-                    }
-
-                    return null;
-                  }}
-                >
-                  <TTDDialogInput
-                    onChange={handleTextChange}
-                    input={text}
-                    placeholder={"Describe what you want to see..."}
-                    onKeyboardSubmit={() => {
-                      refOnGenerate.current();
-                    }}
-                  />
-                </TTDDialogPanel>
-                <TTDDialogPanel
-                  label="Preview"
-                  panelAction={{
-                    action: () => {
-                      console.info("Panel action clicked");
-                      insertToEditor({ app, data });
-                    },
-                    label: "Insert",
-                    icon: ArrowRightIcon,
-                  }}
-                >
-                  <TTDDialogOutput
-                    canvasRef={someRandomDivRef}
-                    error={error}
-                    loaded={mermaidToExcalidrawLib.loaded}
-                  />
-                </TTDDialogPanel>
-              </TTDDialogPanels>
+              <TextToDiagram
+                onTextSubmit={rest.onTextSubmit}
+                mermaidToExcalidrawLib={mermaidToExcalidrawLib}
+              />
+            </TTDDialogTab>
+          )}
+          {!("__fallback" in rest) && (
+            <TTDDialogTab className="ttd-dialog-content" tab="text-to-drawing">
+              <TextToDrawing onTextSubmit={rest.onTextSubmit} />
             </TTDDialogTab>
           )}
         </TTDDialogTabs>

src/components/TTDDialog/TTDDialogTabs.tsx

@@ -7,8 +7,11 @@ const TTDDialogTabs = (
   props: {
     children: ReactNode;
   } & (
-    | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
-    | { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
+    | { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" | "text-to-drawing" }
+    | {
+        dialog: "settings";
+        tab: "text-to-diagram" | "diagram-to-code";
+      }
   ),
 ) => {
   const setAppState = useExcalidrawSetAppState();

src/components/TTDDialog/TTDDialogTrigger.tsx

@@ -9,9 +9,11 @@ import { trackEvent } from "../../analytics";
 export const TTDDialogTrigger = ({
   children,
   icon,
+  tab,
 }: {
   children?: ReactNode;
   icon?: JSX.Element;
+  tab?: string;
 }) => {
   const { TTDDialogTriggerTunnel } = useTunnels();
   const setAppState = useExcalidrawSetAppState();
@@ -21,7 +23,9 @@ export const TTDDialogTrigger = ({
       <DropdownMenu.Item
         onSelect={() => {
           trackEvent("ai", "dialog open", "ttd");
-          setAppState({ openDialog: { name: "ttd", tab: "text-to-diagram" } });
+          setAppState({
+            openDialog: { name: "ttd", tab: tab ?? "text-to-diagram" },
+          });
         }}
         icon={icon ?? brainIcon}
       >

src/components/TTDDialog/TextToDiagram.tsx

@@ -0,0 +1,228 @@
+import { useAtom } from "jotai";
+import { useRef, useState, ChangeEventHandler } from "react";
+import { trackEvent } from "../../analytics";
+import { t } from "../../i18n";
+import { isFiniteNumber } from "../../utils";
+import { ArrowRightIcon } from "../icons";
+import { TTDDialogInput } from "./TTDDialogInput";
+import { TTDDialogOutput } from "./TTDDialogOutput";
+import { TTDDialogPanel } from "./TTDDialogPanel";
+import { TTDDialogPanels } from "./TTDDialogPanels";
+import {
+  CommonDialogProps,
+  MAX_PROMPT_LENGTH,
+  MIN_PROMPT_LENGTH,
+  MermaidToExcalidrawLibProps,
+  convertMermaidToExcalidraw,
+  insertToEditor,
+  rateLimitsAtom,
+  saveMermaidDataToStorage,
+} from "./common";
+import { useApp } from "../App";
+import { NonDeletedExcalidrawElement } from "../../element/types";
+import { BinaryFiles } from "../../types";
+
+export type TextToDiagramProps = CommonDialogProps & {
+  mermaidToExcalidrawLib: MermaidToExcalidrawLibProps;
+};
+
+export const TextToDiagram = ({
+  onTextSubmit,
+  mermaidToExcalidrawLib,
+}: TextToDiagramProps) => {
+  const app = useApp();
+
+  const someRandomDivRef = useRef<HTMLDivElement>(null);
+
+  const [text, setText] = useState("");
+
+  const prompt = text.trim();
+
+  const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (event) => {
+    setText(event.target.value);
+  };
+
+  const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
+  const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
+
+  const data = useRef<{
+    elements: readonly NonDeletedExcalidrawElement[];
+    files: BinaryFiles | null;
+  }>({ elements: [], files: null });
+
+  const [error, setError] = useState<Error | null>(null);
+
+  const onGenerate = async () => {
+    if (
+      prompt.length > MAX_PROMPT_LENGTH ||
+      prompt.length < MIN_PROMPT_LENGTH ||
+      onTextSubmitInProgess ||
+      rateLimits?.rateLimitRemaining === 0
+    ) {
+      if (prompt.length < MIN_PROMPT_LENGTH) {
+        setError(
+          new Error(
+            `Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
+          ),
+        );
+      }
+      if (prompt.length > MAX_PROMPT_LENGTH) {
+        setError(
+          new Error(`Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`),
+        );
+      }
+
+      return;
+    }
+
+    try {
+      setOnTextSubmitInProgess(true);
+
+      trackEvent("ai", "generate", "ttd");
+
+      const { generatedResponse, error, rateLimit, rateLimitRemaining } =
+        await onTextSubmit(prompt, "text-to-diagram");
+
+      if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
+        setRateLimits({ rateLimit, rateLimitRemaining });
+      }
+
+      if (error) {
+        setError(error);
+        return;
+      }
+      if (!generatedResponse) {
+        setError(new Error("Generation failed"));
+        return;
+      }
+
+      try {
+        await convertMermaidToExcalidraw({
+          canvasRef: someRandomDivRef,
+          data,
+          mermaidToExcalidrawLib,
+          setError,
+          mermaidDefinition: generatedResponse,
+        });
+        trackEvent("ai", "mermaid parse success", "ttd");
+        saveMermaidDataToStorage(generatedResponse);
+      } catch (error: any) {
+        console.info(
+          `%cTTD mermaid render errror: ${error.message}`,
+          "color: red",
+        );
+        console.info(
+          `>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\nTTD mermaid definition render errror: ${error.message}`,
+          "color: yellow",
+        );
+        trackEvent("ai", "mermaid parse failed", "ttd");
+        setError(
+          new Error(
+            "Generated an invalid diagram :(. You may also try a different prompt.",
+          ),
+        );
+      }
+    } catch (error: any) {
+      let message: string | undefined = error.message;
+      if (!message || message === "Failed to fetch") {
+        message = "Request failed";
+      }
+      setError(new Error(message));
+    } finally {
+      setOnTextSubmitInProgess(false);
+    }
+  };
+
+  const refOnGenerate = useRef(onGenerate);
+  refOnGenerate.current = onGenerate;
+
+  return (
+    <>
+      <div className="ttd-dialog-desc">
+        Currently we use Mermaid as a middle step, so you'll get best results if
+        you describe a diagram, workflow, flow chart, and similar.
+      </div>
+      <TTDDialogPanels>
+        <TTDDialogPanel
+          label={t("labels.prompt")}
+          panelAction={{
+            action: onGenerate,
+            label: "Generate",
+            icon: ArrowRightIcon,
+          }}
+          onTextSubmitInProgess={onTextSubmitInProgess}
+          panelActionDisabled={
+            prompt.length > MAX_PROMPT_LENGTH ||
+            rateLimits?.rateLimitRemaining === 0
+          }
+          renderTopRight={() => {
+            if (!rateLimits) {
+              return null;
+            }
+
+            return (
+              <div
+                className="ttd-dialog-rate-limit"
+                style={{
+                  fontSize: 12,
+                  marginLeft: "auto",
+                  color:
+                    rateLimits.rateLimitRemaining === 0
+                      ? "var(--color-danger)"
+                      : undefined,
+                }}
+              >
+                {rateLimits.rateLimitRemaining} requests left today
+              </div>
+            );
+          }}
+          renderBottomRight={() => {
+            const ratio = prompt.length / MAX_PROMPT_LENGTH;
+            if (ratio > 0.8) {
+              return (
+                <div
+                  style={{
+                    marginLeft: "auto",
+                    fontSize: 12,
+                    fontFamily: "monospace",
+                    color: ratio > 1 ? "var(--color-danger)" : undefined,
+                  }}
+                >
+                  Length: {prompt.length}/{MAX_PROMPT_LENGTH}
+                </div>
+              );
+            }
+
+            return null;
+          }}
+        >
+          <TTDDialogInput
+            onChange={handleTextChange}
+            input={text}
+            placeholder={"Describe what you want to see..."}
+            onKeyboardSubmit={() => {
+              refOnGenerate.current();
+            }}
+          />
+        </TTDDialogPanel>
+        <TTDDialogPanel
+          label="Preview"
+          panelAction={{
+            action: () => {
+              console.info("Panel action clicked");
+              insertToEditor({ app, data: data.current });
+            },
+            label: "Insert",
+            icon: ArrowRightIcon,
+          }}
+        >
+          <TTDDialogOutput
+            canvasRef={someRandomDivRef}
+            error={error}
+            loaded={mermaidToExcalidrawLib.loaded}
+          />
+        </TTDDialogPanel>
+      </TTDDialogPanels>
+    </>
+  );
+};

src/components/TTDDialog/TextToDrawing.tsx

@@ -0,0 +1,248 @@
+import { useAtom } from "jotai";
+import { useRef, useState, ChangeEventHandler } from "react";
+import { trackEvent } from "../../analytics";
+import { NonDeletedExcalidrawElement } from "../../element/types";
+import { t } from "../../i18n";
+import { isFiniteNumber } from "../../utils";
+import { useApp } from "../App";
+import { ArrowRightIcon } from "../icons";
+import { TTDDialogInput } from "./TTDDialogInput";
+import { TTDDialogOutput } from "./TTDDialogOutput";
+import { TTDDialogPanel } from "./TTDDialogPanel";
+import { TTDDialogPanels } from "./TTDDialogPanels";
+import {
+  CommonDialogProps,
+  MAX_PROMPT_LENGTH,
+  MIN_PROMPT_LENGTH,
+  insertToEditor,
+  rateLimitsAtom,
+  resetPreview,
+} from "./common";
+import {
+  convertToExcalidrawElements,
+  exportToCanvas,
+} from "../../packages/excalidraw/index";
+import { DEFAULT_EXPORT_PADDING } from "../../constants";
+import { canvasToBlob } from "../../data/blob";
+
+export type TextToDrawingProps = CommonDialogProps;
+
+export const TextToDrawing = ({ onTextSubmit }: TextToDrawingProps) => {
+  const app = useApp();
+  const containerRef = useRef<HTMLDivElement>(null);
+
+  const [text, setText] = useState("");
+
+  const prompt = text.trim();
+
+  const handleTextChange: ChangeEventHandler<HTMLTextAreaElement> = (event) => {
+    setText(event.target.value);
+  };
+
+  const [onTextSubmitInProgess, setOnTextSubmitInProgess] = useState(false);
+  const [rateLimits, setRateLimits] = useAtom(rateLimitsAtom);
+
+  const [data, setData] = useState<
+    readonly NonDeletedExcalidrawElement[] | null
+  >(null);
+
+  const [error, setError] = useState<Error | null>(null);
+
+  const onGenerate = async () => {
+    if (
+      prompt.length > MAX_PROMPT_LENGTH ||
+      prompt.length < MIN_PROMPT_LENGTH ||
+      onTextSubmitInProgess ||
+      rateLimits?.rateLimitRemaining === 0
+    ) {
+      if (prompt.length < MIN_PROMPT_LENGTH) {
+        setError(
+          new Error(
+            `Prompt is too short (min ${MIN_PROMPT_LENGTH} characters)`,
+          ),
+        );
+      }
+      if (prompt.length > MAX_PROMPT_LENGTH) {
+        setError(
+          new Error(`Prompt is too long (max ${MAX_PROMPT_LENGTH} characters)`),
+        );
+      }
+
+      return;
+    }
+
+    try {
+      setOnTextSubmitInProgess(true);
+
+      trackEvent("ai", "generate", "text-to-drawing");
+
+      const { generatedResponse, error, rateLimit, rateLimitRemaining } =
+        await onTextSubmit(prompt, "text-to-drawing");
+
+      if (isFiniteNumber(rateLimit) && isFiniteNumber(rateLimitRemaining)) {
+        setRateLimits({ rateLimit, rateLimitRemaining });
+      }
+
+      if (error) {
+        setError(error);
+        return;
+      }
+      if (!generatedResponse) {
+        setError(new Error("Generation failed"));
+        return;
+      }
+
+      const canvasNode = containerRef.current;
+      const parent = canvasNode?.parentElement;
+
+      if (!canvasNode || !parent) {
+        return;
+      }
+
+      if (!text) {
+        resetPreview({ canvasRef: containerRef, setError });
+        return;
+      }
+
+      if (!Array.isArray(generatedResponse)) {
+        setError(new Error("Generation failed to return an array!"));
+        return;
+      }
+
+      try {
+        const elements = convertToExcalidrawElements(generatedResponse, {
+          regenerateIds: true,
+        });
+
+        setData(elements);
+
+        const canvas = await exportToCanvas({
+          elements,
+          files: null,
+          exportPadding: DEFAULT_EXPORT_PADDING,
+          maxWidthOrHeight:
+            Math.max(parent.offsetWidth, parent.offsetHeight) *
+            window.devicePixelRatio,
+        });
+        // if converting to blob fails, there's some problem that will
+        // likely prevent preview and export (e.g. canvas too big)
+        await canvasToBlob(canvas);
+        parent.style.background = "var(--default-bg-color)";
+        canvasNode.replaceChildren(canvas);
+      } catch (err: any) {
+        console.error(err);
+        parent.style.background = "var(--default-bg-color)";
+        if (text) {
+          setError(err);
+        }
+
+        throw err;
+      }
+    } catch (error: any) {
+      let message: string | undefined = error.message;
+      if (!message || message === "Failed to fetch") {
+        message = "Request failed";
+      }
+      setError(new Error(message));
+    } finally {
+      setOnTextSubmitInProgess(false);
+    }
+  };
+
+  const refOnGenerate = useRef(onGenerate);
+  refOnGenerate.current = onGenerate;
+
+  return (
+    <>
+      <div className="ttd-dialog-desc">This is text to drawing.</div>
+      <TTDDialogPanels>
+        <TTDDialogPanel
+          label={t("labels.prompt")}
+          panelAction={{
+            action: onGenerate,
+            label: "Generate",
+            icon: ArrowRightIcon,
+          }}
+          onTextSubmitInProgess={onTextSubmitInProgess}
+          panelActionDisabled={
+            prompt.length > MAX_PROMPT_LENGTH ||
+            rateLimits?.rateLimitRemaining === 0
+          }
+          renderTopRight={() => {
+            if (!rateLimits) {
+              return null;
+            }
+
+            return (
+              <div
+                className="ttd-dialog-rate-limit"
+                style={{
+                  fontSize: 12,
+                  marginLeft: "auto",
+                  color:
+                    rateLimits.rateLimitRemaining === 0
+                      ? "var(--color-danger)"
+                      : undefined,
+                }}
+              >
+                {rateLimits.rateLimitRemaining} requests left today
+              </div>
+            );
+          }}
+          renderBottomRight={() => {
+            const ratio = prompt.length / MAX_PROMPT_LENGTH;
+            if (ratio > 0.8) {
+              return (
+                <div
+                  style={{
+                    marginLeft: "auto",
+                    fontSize: 12,
+                    fontFamily: "monospace",
+                    color: ratio > 1 ? "var(--color-danger)" : undefined,
+                  }}
+                >
+                  Length: {prompt.length}/{MAX_PROMPT_LENGTH}
+                </div>
+              );
+            }
+
+            return null;
+          }}
+        >
+          <TTDDialogInput
+            onChange={handleTextChange}
+            input={text}
+            placeholder={"Describe what you want to see..."}
+            onKeyboardSubmit={() => {
+              refOnGenerate.current();
+            }}
+          />
+        </TTDDialogPanel>
+        <TTDDialogPanel
+          label="Preview"
+          panelAction={{
+            action: () => {
+              if (data) {
+                insertToEditor({
+                  app,
+                  data: {
+                    elements: data,
+                    files: null,
+                  },
+                });
+              }
+            },
+            label: "Insert",
+            icon: ArrowRightIcon,
+          }}
+        >
+          <TTDDialogOutput
+            canvasRef={containerRef}
+            error={error}
+            loaded={true}
+          />
+        </TTDDialogPanel>
+      </TTDDialogPanels>
+    </>
+  );
+};

src/components/TTDDialog/common.ts

@@ -8,8 +8,9 @@ import {
 import { NonDeletedExcalidrawElement } from "../../element/types";
 import { AppClassProperties, BinaryFiles } from "../../types";
 import { canvasToBlob } from "../../data/blob";
+import { atom } from "jotai";
 
-const resetPreview = ({
+export const resetPreview = ({
   canvasRef,
   setError,
 }: {
@@ -30,6 +31,26 @@ const resetPreview = ({
   canvasNode.replaceChildren();
 };
 
+export type OnTestSubmitRetValue = {
+  rateLimit?: number | null;
+  rateLimitRemaining?: number | null;
+} & (
+  | {
+      generatedResponse: any | string | undefined;
+      error?: null | undefined;
+    }
+  | {
+      error: Error;
+      generatedResponse?: null | undefined;
+    }
+);
+export interface CommonDialogProps {
+  onTextSubmit(
+    value: string,
+    type: "text-to-diagram" | "text-to-drawing",
+  ): Promise<OnTestSubmitRetValue>;
+}
+
 export interface MermaidToExcalidrawLibProps {
   loaded: boolean;
   api: Promise<{
@@ -137,14 +158,14 @@ export const insertToEditor = ({
   shouldSaveMermaidDataToStorage,
 }: {
   app: AppClassProperties;
-  data: React.MutableRefObject<{
+  data: {
     elements: readonly NonDeletedExcalidrawElement[];
     files: BinaryFiles | null;
-  }>;
+  };
   text?: string;
   shouldSaveMermaidDataToStorage?: boolean;
 }) => {
-  const { elements: newElements, files } = data.current;
+  const { elements: newElements, files } = data;
 
   if (!newElements.length) {
     return;
@@ -162,3 +183,11 @@ export const insertToEditor = ({
     saveMermaidDataToStorage(text);
   }
 };
+
+export const MIN_PROMPT_LENGTH = 3;
+export const MAX_PROMPT_LENGTH = 1000;
+
+export const rateLimitsAtom = atom<{
+  rateLimit: number;
+  rateLimitRemaining: number;
+} | null>(null);

src/components/TextToExcalidraw/TextToExcalidraw.tsx

@@ -1,589 +0,0 @@
-import { useEffect, useRef, useState } from "react";
-import { t } from "../../i18n";
-import { useApp } from "../App";
-import { Dialog } from "../Dialog";
-import { TextField } from "../TextField";
-import Trans from "../Trans";
-import {
-  CloseIcon,
-  RedoIcon,
-  ZoomInIcon,
-  ZoomOutIcon,
-  playerPlayIcon,
-  playerStopFilledIcon,
-} from "../icons";
-import { NonDeletedExcalidrawElement } from "../../element/types";
-import { convertToExcalidrawElements } from "../../data/transform";
-import { exportToCanvas } from "../../packages/utils";
-import { DEFAULT_EXPORT_PADDING } from "../../constants";
-import { canvasToBlob } from "../../data/blob";
-
-const testResponse = `{
-  "error": false,
-  "data": [
-    {
-      "type": "ellipse",
-      "x": 200,
-      "y": 200,
-      "width": 100,
-      "height": 100,
-      "strokeColor": "transparent",
-      "backgroundColor": "yellow",
-      "strokeWidth": 2
-    },
-    {
-      "type": "line",
-      "x": 300,
-      "y": 250,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          70,
-          0
-        ]
-      ],
-      "width": -70,
-      "height": 0,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 293.30127018922195,
-      "y": 275,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          60.62177826491069,
-          35
-        ]
-      ],
-      "width": -60.62177826491069,
-      "height": -35,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 275,
-      "y": 293.30127018922195,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          35,
-          60.62177826491069
-        ]
-      ],
-      "width": -35,
-      "height": -60.62177826491069,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 250,
-      "y": 300,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          0,
-          70
-        ]
-      ],
-      "width": 0,
-      "height": -70,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 225,
-      "y": 293.30127018922195,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -34.99999999999997,
-          60.62177826491069
-        ]
-      ],
-      "width": -34.99999999999997,
-      "height": -60.62177826491069,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 206.69872981077805,
-      "y": 275,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -60.62177826491069,
-          35
-        ]
-      ],
-      "width": -60.62177826491069,
-      "height": -35,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 200,
-      "y": 250,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -70,
-          2.842170943040401e-14
-        ]
-      ],
-      "width": -70,
-      "height": -2.842170943040401e-14,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 206.69872981077805,
-      "y": 225,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -60.62177826491069,
-          -34.99999999999997
-        ]
-      ],
-      "width": -60.62177826491069,
-      "height": -34.99999999999997,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 224.99999999999997,
-      "y": 206.69872981077808,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -35.00000000000003,
-          -60.62177826491069
-        ]
-      ],
-      "width": -35.00000000000003,
-      "height": -60.62177826491069,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 250,
-      "y": 200,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          -2.842170943040401e-14,
-          -70
-        ]
-      ],
-      "width": -2.842170943040401e-14,
-      "height": -70,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 275,
-      "y": 206.69872981077808,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          35,
-          -60.621778264910716
-        ]
-      ],
-      "width": -35,
-      "height": -60.621778264910716,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    },
-    {
-      "type": "line",
-      "x": 293.3012701892219,
-      "y": 224.99999999999997,
-      "points": [
-        [
-          0,
-          0
-        ],
-        [
-          60.621778264910745,
-          -35.00000000000003
-        ]
-      ],
-      "width": -60.621778264910745,
-      "height": -35.00000000000003,
-      "strokeColor": "yellow",
-      "backgroundColor": "transparent",
-      "strokeWidth": 5
-    }
-  ]
-}`;
-
-async function fetchData(
-  prompt: string,
-): Promise<readonly NonDeletedExcalidrawElement[]> {
-  const response = await fetch(
-    `http://localhost:3015/v1/ai/text-to-excalidraw/generate`,
-    {
-      method: "POST",
-      headers: {
-        Accept: "application/json",
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({ prompt }),
-    },
-  );
-
-  const result = await response.json();
-
-  if (result.error) {
-    alert("Oops!");
-    return [];
-  }
-
-  return convertToExcalidrawElements(result.data);
-}
-
-export const TextToExcalidraw = () => {
-  const app = useApp();
-
-  const [prompt, setPrompt] = useState("");
-  const [isPanelOpen, setPanelOpen] = useState(false);
-  const [isLoading, setLoading] = useState(false);
-  const [data, setData] = useState<
-    readonly NonDeletedExcalidrawElement[] | null
-  >(null);
-
-  const [previewCanvas, setPreviewCanvas] = useState<HTMLCanvasElement | null>(
-    null,
-  );
-
-  const containerRef = useRef<HTMLDivElement>(null);
-
-  const onClose = () => {
-    app.setOpenDialog(null);
-  };
-
-  const onSubmit = async () => {
-    setPanelOpen(true);
-    setLoading(true);
-
-    const elements = await fetchData(prompt);
-
-    setData(elements);
-
-    const canvas = await exportToCanvas({
-      elements,
-      files: {},
-      exportPadding: DEFAULT_EXPORT_PADDING,
-    });
-
-    await canvasToBlob(canvas);
-
-    setPreviewCanvas(canvas);
-    setLoading(false);
-  };
-
-  const onInsert = async () => {
-    if (data) {
-      app.addElementsFromPasteOrLibrary({
-        elements: data,
-        files: {},
-        position: "center",
-        fitToContent: true,
-      });
-
-      onClose();
-    }
-  };
-
-  useEffect(() => {
-    if (containerRef.current && previewCanvas) {
-      containerRef.current.replaceChildren(previewCanvas);
-    }
-  }, [previewCanvas]);
-
-  // exportToCanvas([], {}, {}, {});
-  // exportToSvg([], {exportBackground}, {}, {})
-
-  return (
-    <div
-      style={{
-        position: "absolute",
-        top: "6.5rem",
-        pointerEvents: "auto",
-        width: "100%",
-        display: "flex",
-        flexDirection: "column",
-        gap: "0.75rem",
-      }}
-    >
-      <div
-        className="Island"
-        style={{
-          width: "100%",
-          display: "flex",
-          flexDirection: "row",
-          boxSizing: "border-box",
-          gap: "0.75rem",
-          alignItems: "center",
-          height: 48,
-          padding: "0.5rem",
-        }}
-      >
-        <input
-          value={prompt}
-          onChange={(e) => setPrompt(e.target.value)}
-          type="text"
-          style={{
-            flexGrow: 1,
-            height: "100%",
-            boxSizing: "border-box",
-            border: 0,
-            outline: "none",
-          }}
-          placeholder="How can I help you today?"
-        />
-        <button
-          style={{
-            cursor: "pointer",
-            height: "100%",
-            border: "none",
-            background: "white",
-            aspectRatio: "1/1",
-            padding: 0,
-            display: "flex",
-            justifyContent: "center",
-            alignItems: "center",
-          }}
-        >
-          <div
-            style={{ width: "1.25rem", height: "1.25rem", color: "#1B1B1F" }}
-          >
-            {CloseIcon}
-          </div>
-        </button>
-        <div
-          style={{ background: "#D6D6D6", width: 1, height: "1.5rem" }}
-        ></div>
-        <button
-          style={{
-            cursor: "pointer",
-            height: "100%",
-            border: "none",
-            aspectRatio: "1/1",
-            padding: 0,
-            display: "flex",
-            justifyContent: "center",
-            alignItems: "center",
-            backgroundColor: "#6965DB",
-            borderRadius: "0.5rem",
-          }}
-          onClick={onSubmit}
-        >
-          <div style={{ width: "1.25rem", height: "1.25rem", color: "white" }}>
-            {isLoading ? playerStopFilledIcon : playerPlayIcon}
-          </div>
-        </button>
-      </div>
-
-      {isPanelOpen && (
-        <div
-          className="Island"
-          style={{
-            width: "100%",
-            display: "flex",
-            flexDirection: "row",
-            height: 400,
-            boxSizing: "border-box",
-          }}
-        >
-          {isLoading ? (
-            "loading"
-          ) : (
-            <div
-              style={{
-                width: "100%",
-                height: "100%",
-                display: "flex",
-                flexDirection: "column",
-              }}
-            >
-              <div
-                ref={containerRef}
-                style={{
-                  display: "flex",
-                  alignItems: "center",
-                  justifyContent: "center",
-                  width: "100%",
-                  height: "100%",
-                }}
-              />
-              <div
-                style={{
-                  borderTop: "1px solid #F0EFFF",
-                  padding: "0.75rem",
-                  display: "flex",
-                  flexDirection: "row",
-                  gap: "0.75rem",
-                }}
-              >
-                <button
-                  style={{
-                    cursor: "pointer",
-                    width: 32,
-                    height: "100%",
-                    border: "none",
-                    aspectRatio: "1/1",
-                    padding: 0,
-                    display: "flex",
-                    justifyContent: "center",
-                    alignItems: "center",
-                    backgroundColor: "#F5F5F9",
-                    borderRadius: "0.5rem",
-                  }}
-                >
-                  <div
-                    style={{
-                      width: "1.25rem",
-                      height: "1.25rem",
-                      color: "#1B1B1F",
-                    }}
-                  >
-                    {RedoIcon}
-                  </div>
-                </button>
-
-                <div style={{ width: 32, height: "100%", display: "flex" }}>
-                  <button
-                    style={{
-                      cursor: "pointer",
-                      width: 32,
-                      height: "100%",
-                      border: "none",
-                      aspectRatio: "1/1",
-                      padding: 0,
-                      display: "flex",
-                      justifyContent: "center",
-                      alignItems: "center",
-                      backgroundColor: "#F5F5F9",
-                      borderRadius: "0.5rem 0 0 0.5rem",
-                    }}
-                  >
-                    <div
-                      style={{
-                        width: "1.25rem",
-                        height: "1.25rem",
-                        color: "#1B1B1F",
-                      }}
-                    >
-                      {ZoomOutIcon}
-                    </div>
-                  </button>
-                  <button
-                    style={{
-                      cursor: "pointer",
-                      width: 32,
-                      height: "100%",
-                      border: "none",
-                      aspectRatio: "1/1",
-                      padding: 0,
-                      display: "flex",
-                      justifyContent: "center",
-                      alignItems: "center",
-                      backgroundColor: "#F5F5F9",
-                      borderRadius: "0 0.5rem 0.5rem 0",
-                    }}
-                  >
-                    <div
-                      style={{
-                        width: "1.25rem",
-                        height: "1.25rem",
-                        color: "#1B1B1F",
-                      }}
-                    >
-                      {ZoomInIcon}
-                    </div>
-                  </button>
-                </div>
-                <div style={{ flexGrow: 1 }}></div>
-                <button
-                  style={{
-                    cursor: "pointer",
-                    height: "100%",
-                    border: "none",
-                    padding: "0.5rem 1rem",
-                    display: "flex",
-                    justifyContent: "center",
-                    alignItems: "center",
-                    backgroundColor: "#6965DB",
-                    borderRadius: "0.5rem",
-                    color: "white",
-                  }}
-                  onClick={onInsert}
-                >
-                  Insert into scene &gt;
-                </button>
-              </div>
-            </div>
-          )}
-        </div>
-      )}
-    </div>
-  );
-};

src/components/icons.tsx

@@ -1755,3 +1755,13 @@ export const brainIcon = createIcon(
   </g>,
   tablerIconProps,
 );
+
+export const drawingIcon = createIcon(
+  <g stroke="currentColor" fill="none">
+    <path stroke="none" d="M0 0h24v24H0z" fill="none" />
+    <path d="M20 17v-12c0 -1.121 -.879 -2 -2 -2s-2 .879 -2 2v12l2 2l2 -2z" />
+    <path d="M16 7h4" />
+    <path d="M18 19h-13a2 2 0 1 1 0 -4h4a2 2 0 1 0 0 -4h-3" />
+  </g>,
+  tablerIconProps,
+);

src/locales/en.json

@@ -134,7 +134,8 @@
     "removeAllElementsFromFrame": "Remove all elements from frame",
     "eyeDropper": "Pick color from canvas",
     "textToDiagram": "Text to diagram",
-    "prompt": "Prompt"
+    "prompt": "Prompt",
+    "textToDrawing": "Text to drawing"
   },
   "library": {
     "noItems": "No items added yet...",