Google Gemini 1.5 Pro & Flash API: Multimodal Prompting, Function Calling & Structured JSON with Node.js

npm install @google/generative-ai dotenv

GEMINI_API_KEY=your_actual_gemini_api_key_here

import { GoogleGenerativeAI } from "@google/generative-ai";

const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);

// Gemini 1.5 Flash: Ultra-fast, lightweight, highly cost-effective
export const flashModel = genAI.getGenerativeModel({
  model: "gemini-1.5-flash",
  generationConfig: {
    temperature: 0.2, // Low temperature for deterministic outputs
    topP: 0.8,
    maxOutputTokens: 2048,
  },
});

// Gemini 1.5 Pro: Deep reasoning, complex coding, and massive document analysis
export const proModel = genAI.getGenerativeModel({
  model: "gemini-1.5-pro",
});

import * as fs from "fs";

function fileToGenerativePart(path: string, mimeType: string) {
  return {
    inlineData: {
      data: Buffer.from(fs.readFileSync(path)).toString("base64"),
      mimeType,
    },
  };
}

export async function analyzeArchitectureDiagram(imagePath: string) {
  const imagePart = fileToGenerativePart(imagePath, "image/png");

  const prompt = `
    Analyze this system architecture diagram.
    1. Identify all database and caching nodes.
    2. Point out potential single points of failure.
    3. Suggest architectural resilience improvements.
  `;

  const result = await flashModel.generateContent([prompt, imagePart]);
  return result.response.text();
}

import { SchemaType } from "@google/generative-ai";

const structuredModel = genAI.getGenerativeModel({
  model: "gemini-1.5-flash",
  generationConfig: {
    responseMimeType: "application/json",
    responseSchema: {
      type: SchemaType.OBJECT,
      properties: {
        recipeName: { type: SchemaType.STRING },
        prepTimeMinutes: { type: SchemaType.NUMBER },
        ingredients: {
          type: SchemaType.ARRAY,
          items: { type: SchemaType.STRING },
        },
        difficulty: {
          type: SchemaType.STRING,
          enum: ["Easy", "Medium", "Hard"],
        },
      },
      required: ["recipeName", "prepTimeMinutes", "ingredients", "difficulty"],
    },
  },
});

export async function generateStructuredRecipe(userPrompt: string) {
  const result = await structuredModel.generateContent(userPrompt);
  // Guaranteed valid JSON matching the schema!
  const parsedData = JSON.parse(result.response.text());
  return parsedData;
}

const databaseTool = {
  functionDeclarations: [
    {
      name: "getUserSubscriptionStatus",
      description: "Lookup a user's current billing tier and active credits",
      parameters: {
        type: SchemaType.OBJECT,
        properties: {
          userId: { type: SchemaType.STRING, description: "The unique customer ID" },
        },
        required: ["userId"],
      },
    },
  ],
};

const toolModel = genAI.getGenerativeModel({
  model: "gemini-1.5-flash",
  tools: [databaseTool],
});

export async function handleCustomerInquiry(userMessage: string) {
  const chat = toolModel.startChat();
  const response = await chat.sendMessage(userMessage);

  const functionCalls = response.response.functionCalls();
  if (functionCalls && functionCalls.length > 0) {
    const call = functionCalls[0];
    console.log(`Gemini requested tool execution: ${call.name}`, call.args);

    // Execute actual database query
    const dbResult = { tier: "Enterprise", activeCredits: 4500 };

    // Send function execution result back to Gemini
    const finalResponse = await chat.sendMessage([
      {
        functionResponse: {
          name: call.name,
          response: dbResult,
        },
      },
    ]);

    return finalResponse.response.text();
  }

  return response.response.text();
}