Embeddings - Portkey Docs

Vertex AI offers wide ranging support for embedding text, images and videos. Portkey provides a standardized interface for embedding multiple modalities.

Embedding Text

from portkey_ai import Portkey

client = Portkey(
    api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
    provider="@PROVIDER",
)

embeddings = client.embeddings.create(
  model="textembedding-gecko@003",
  input_type="classification",
  input="The food was delicious and the waiter...",
  # input=["text to embed", "more text to embed"], # if you would like to embed multiple texts
)

import { Portkey } from 'portkey-ai';

const portkey = new Portkey({
    apiKey: "YOUR_API_KEY",
    provider:"@YOUR_PROVIDER"
});

const embedding = await portkey.embeddings.create({
    input: 'Name the tallest buildings in Hawaii',
    // input: ['text to embed', 'more text to embed'], // if you would like to embed multiple texts
    model: 'textembedding-gecko@003'
});

console.log(embedding);

curl --location 'https://api.portkey.ai/v1/embeddings' \
--header 'Content-Type: application/json' \
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
--header 'x-portkey-provider: PORTKEY_PROVIDER' \
--data-raw '{
    "model": "textembedding-gecko@003",
    "input": [
        "A HTTP 246 code is used to signify an AI response containing hallucinations or other inaccuracies",
        "246: Partially incorrect response"
    ],
    # "input": "Name the tallest buildings in Hawaii",
    "input_type": "classification"
}'

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

portkey_client = OpenAI(
    api_key='NOT_REQUIRED',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

embeddings = portkey_client.embeddings.create(
  model="textembedding-gecko@003",
  input_type="classification",
  input="The food was delicious and the waiter...",
  # input=["text to embed", "more text to embed"], # if you would like to embed multiple texts
)

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const portkeyClient = new OpenAI({
  apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "vertex-ai",
    apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
    provider:"@PORTKEY_PROVIDER"
  })
});

const embedding = await portkeyClient.embeddings.create({
    input: 'Name the tallest buildings in Hawaii',
    // input: ['text to embed', 'more text to embed'], // if you would like to embed multiple texts
    model: 'textembedding-gecko@003'
});

console.log(embedding);

Embeddings Images

from portkey_ai import Portkey

client = Portkey(
    api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
    provider="@PROVIDER",
)

embeddings = client.embeddings.create(
  model="multimodalembedding@001",
  input=[
          {
              "text": "this is the caption of the image",
              "image": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  # "url": "gcs://..." # if you want to use a url
              }
          }
      ]
)

import { Portkey } from 'portkey-ai';

const portkey = new Portkey({
    apiKey: "YOUR_API_KEY",
    provider:"@YOUR_PROVIDER"
});

const embedding = await portkey.embeddings.create({
    input: [
          {
              "text": "this is the caption of the image",
              "image": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  // "url": "gcs://..." // if you want to use a url
              }
          }
      ],
    model: 'multimodalembedding@001'
});

console.log(embedding);

curl --location 'https://api.portkey.ai/v1/embeddings' \
--header 'Content-Type: application/json' \
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
--header 'x-portkey-provider: PORTKEY_PROVIDER' \
--data-raw '{
    "model": "multimodalembedding@001",
    "input": [
                  {
                      "text": "this is the caption of the image",
                      "image": {
                          "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B....."
                          # "url": "gcs://..." # if you want to use a url
                      }
                  }
              ]
}'

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

portkey_client = OpenAI(
    api_key='NOT_REQUIRED',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

embeddings = portkey_client.embeddings.create(
  model="multimodalembedding@001",
  input=[
          {
              "text": "this is the caption of the image",
              "image": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  # "url": "gcs://..." # if you want to use a url
              }
          }
        ]
)

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const portkeyClient = new OpenAI({
  apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
    provider:"@PORTKEY_PROVIDER"
  })
});

const embedding = await portkeyClient.embeddings.create({
    input: [
              {
                  "text": "this is the caption of the image",
                  "image": {
                      "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                      // "url": "gcs://..." // if you want to use a url
                  }
              }
            ],
    model: 'multimodalembedding@001'
});

console.log(embedding);

Embeddings Videos

from portkey_ai import Portkey

client = Portkey(
    api_key="YOUR_PORTKEY_API_KEY", # defaults to os.environ.get("PORTKEY_API_KEY")
    provider="@PROVIDER",
)

embeddings = client.embeddings.create(
  model="multimodalembedding@001",
  input=[
          {
              "text": "this is the caption of the video",
              "video": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  "start_offset": 0,
                  "end_offset": 10,
                  "interval": 5,
                  # "url": "gcs://..." # if you want to use a url
              }
          }
      ]
)

import { Portkey } from 'portkey-ai';

const portkey = new Portkey({
    apiKey: "YOUR_API_KEY",
    provider:"@YOUR_PROVIDER"
});

const embedding = await portkey.embeddings.create({
    input: [
          {
              "text": "this is the caption of the video",
              "video": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  "start_offset": 0,
                  "end_offset": 10,
                  "interval": 5,
                  // "url": "gcs://..." // if you want to use a url
              }
          }
      ],
    model: 'multimodalembedding@001'
});

console.log(embedding);

curl --location 'https://api.portkey.ai/v1/embeddings' \
--header 'Content-Type: application/json' \
--header 'x-portkey-api-key: PORTKEY_API_KEY' \
--header 'x-portkey-provider: PORTKEY_PROVIDER' \
--data-raw '{
    "model": "multimodalembedding@001",
    "input": [
                  {
                      "text": "this is the caption of the video",
                      "video": {
                          "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                          "start_offset": 0,
                          "end_offset": 10,
                          "interval": 5
                          # "url": "gcs://..." # if you want to use a url
                      }
                  }
              ]
}'

from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

portkey_client = OpenAI(
    api_key='NOT_REQUIRED',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

embeddings = portkey_client.embeddings.create(
  model="multimodalembedding@001",
  input=[
          {
              "text": "this is the caption of the video",
              "video": {
                  "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                  "start_offset": 0,
                  "end_offset": 10,
                  "interval": 5,
                  # "url": "gcs://..." # if you want to use a url
              }
          }
        ]
)

import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const portkeyClient = new OpenAI({
  apiKey: 'NOT_REQUIRED', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    apiKey: "PORTKEY_API_KEY", // defaults to process.env["PORTKEY_API_KEY"]
    provider:"@PORTKEY_PROVIDER"
  })
});

const embedding = await portkeyClient.embeddings.create({
    input: [
              {
                  "text": "this is the caption of the video",
                  "video": {
                      "base64": "UklGRkacAABXRUJQVlA4IDqcAACQggKdASqpAn8B.....",
                      "start_offset": 0,
                      "end_offset": 10,
                      "interval": 5,
                      // "url": "gcs://..." // if you want to use a url
                  }
              }
            ],
    model: 'multimodalembedding@001'
});

console.log(embedding);

​Embedding Text

​Embeddings Images

​Embeddings Videos

Embedding Text

Embeddings Images

Embeddings Videos