Semantic Image Based Queries Using LangChain (OpenAI) and Redis

import {
  ChatOpenAI,
  ChatOpenAICallOptions,
} from 'langchain/chat_models/openai';
import { HumanMessage } from 'langchain/schema';
import { Document } from 'langchain/document';
import { OpenAIEmbeddings } from 'langchain/embeddings/openai';
import { RedisVectorStore } from 'langchain/vectorstores/redis';

let llm: ChatOpenAI<ChatOpenAICallOptions>;

// Instantiates the LangChain ChatOpenAI instance
const getOpenAIVisionInstance = (_openAIApiKey: string) => {
  //OpenAI supports  images with text in input messages with their gpt-4-vision-preview.
  if (!llm) {
    llm = new ChatOpenAI({
      openAIApiKey: _openAIApiKey,
      modelName: 'gpt-4-vision-preview',
      maxTokens: 1024,
    });
  }
  return llm;
};

const fetchImageAndConvertToBase64 = async (_imageURL: string) => {
  let base64Image = '';
  try {
    const response = await axios.get(_imageURL, {
      responseType: 'arraybuffer',
    });
    // Convert image to Base64
    base64Image = Buffer.from(response.data, 'binary').toString('base64');
  } catch (error) {
    console.error(
      `Error fetching or converting the image: ${_imageURL}`,
      error,
    );
  }
  return base64Image;
};

// Generates an OpenAI summary for a given base64 image string
const getOpenAIImageSummary = async (
  _openAIApiKey: string,
  _base64Image: string,
  _product: Prisma.ProductCreateInput,
) => {
  /*
     Reference : https://js.langchain.com/docs/integrations/chat/openai#multimodal-messages

    - This function utilizes OpenAI's multimodal capabilities to generate a summary from the image. 
    - It constructs a prompt that combines the product description with the image.
    - OpenAI's vision model then processes this prompt to generate a detailed summary.

   */
  let imageSummary = '';

  try {
    if (_openAIApiKey && _base64Image && _product) {
      const llmInst = getOpenAIVisionInstance(_openAIApiKey);

      const text = `Below are the product details and image of an e-commerce product for reference. Please conduct and provide a comprehensive analysis of the product depicted in the image . 
        
            Product Details:
            ${_product.productDescriptors_description_value}
            
            Image:
        `;
      // Constructing a multimodal message combining text and image
      const imagePromptMessage = new HumanMessage({
        content: [
          {
            type: 'text',
            text: text,
          },
          {
            type: 'image_url',
            image_url: {
              url: `data:image/jpeg;base64,${_base64Image}`,
              detail: 'high', // low, high (if you want more detail)
            },
          },
        ],
      });

      // Invoking the LangChain ChatOpenAI model with the constructed message
      const response = await llmInst.invoke([imagePromptMessage]);
      if (response?.content) {
        imageSummary = <string>response.content;
      }
    }
  } catch (err) {
    console.log(
      `Error generating OpenAIImageSummary for product id ${_product.productId}`,
      err,
    );
  }
  return imageSummary;
};