Automate your
desktop with
Node.js

import {
    mouse, Button, keyboard, straightTo, centerOf, getActiveWindow
} from "@nut-tree/nut-js";
import { useElementInspector } from "@nut-tree/element-inspector";
import { elements } from "@nut-tree/element-inspector/win";

useElementInspector();

describe("Login Flow - UI Elements", () => {
  it("should login successfully", async () => {
    const window = await getActiveWindow();

    // Find login button by UI element
    const loginBtn = await window.find(
      elements.button({ title: "Sign In" })
    );
    await mouse.move(straightTo(centerOf(loginBtn.region)));
    await mouse.click(Button.LEFT);

    // Find input field and enter credentials
    const emailField = await window.find(
      elements.textInput({ id: "email" })
    );
    await mouse.move(straightTo(centerOf(emailField.region)));
    await mouse.click(Button.LEFT);
    await keyboard.type("user@test.com");
  });
});

import {
    screen, mouse, Button, straightTo
} from "@nut-tree/nut-js";

async function executeAction(action: Action) {
    // Capture current screen state
    const screenshot = await screen.capture();

    // Let AI analyze the screen
    const target = await analyzeWithLLM(
        screenshot, action.description
    );

    await mouse.move(straightTo(target.coordinates));
    await mouse.click(Button.LEFT);
}