Having trouble using other models in node.js example.

[johnpg82]

I'm trying to use other models besides openai in your example code but I'm just getting a response back "invalid model". I've only tried with anthropic and perplexity so haven't set any of the other models supported in the array but those two models don't appear to be working. Note: It works fine if I use OpenAI models.

BadRequestError: 400 invalid model ID
    at APIError.generate (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/error.mjs:41:20)
    at OpenAI.makeStatusError (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:286:25)
    at OpenAI.makeRequest (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:330:30)
    at process.processTicksAndRejections (node:internal/process/task_queues:105:5)
    at async main (file:///Users/jgarland/Documents/GitHub/ai-api/src/example.js:59:18) {
  status: 400,
  headers: {
    'alt-svc': 'h3=":443"; ma=86400',
    'cf-cache-status': 'DYNAMIC',
    'cf-ray': '8e2a1db4c819e64d-DEN',
    connection: 'keep-alive',
    'content-length': '149',
    'content-type': 'application/json; charset=utf-8',
    date: 'Thu, 14 Nov 2024 21:32:46 GMT',
    server: 'cloudflare',
    'set-cookie': '__cf_bm=-1731619966-1.0.1.1-.iEwnKF9.cvczeAyw; path=/; expires=Thu, 14-Nov-24 22:02:46 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=-1731619966293-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None',
    'strict-transport-security': 'max-age=31536000; includeSubDomains; preload',
    vary: 'Origin',
    'x-content-type-options': 'nosniff',
    'x-request-id': 'req_957384571a2399d617daa5ec52787002'
  },
  request_id: 'req_957384571a2399d617daa5ec52787002',
  error: {
    message: 'invalid model ID',
    type: 'invalid_request_error',
    param: null,
    code: null
  },
  code: null,
  param: null,
  type: 'invalid_request_error'
}

Here is some sample javascript code I'm using.

import OpenAI from 'openai';
import dotenv from 'dotenv';
dotenv.config({ path: '../.env' });

const modelApiKeys = {
  'openai': {
    apiKey: process.env.OPENAI_API_KEY,
    models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4']
  },
  'anthropic': {
    apiKey: process.env.ANTHROPIC_API_KEY,
    models: ['anthropic.claude-v2']
  },
  'groq': {
    apiKey: process.env.GROQ_API_KEY,
    models: ['groq-model-1', 'groq-model-2']
  },
  'gemini': {
    apiKey: process.env.GEMINI_API_KEY,
    models: ['gemini-model-1', 'gemini-model-2']
  },
  'perplexity': {
    apiKey: process.env.PERPLEXITYAI_API_KEY,
    models: ['perplexity/mixtral-8x7b-instruct', 'perplexity/llama-3-sonar-large-32k-online']
  },
  'cohere': {
    apiKey: process.env.COHERE_API_KEY,
    models: ['cohere-model-1', 'cohere-model-2']
  },
  // Add other models and their corresponding API keys here
};

const model = 'perplexity/llama-3-sonar-large-32k-online'; // Change this to switch between models
//const model = 'gpt-4o-mini'; // Change this to switch between models
let apiKey;

for (const { apiKey: key, models } of Object.values(modelApiKeys)) {
  if (models.includes(model)) {
    apiKey = key;
    break;
  }
}

if (!apiKey) {
  throw new Error(`API key for model ${model} is not defined`);
}

const baseUrl = process.env.base_url || "https://open-assistant-ai.astra.datastax.com/v1";

async function main(apiKey, model) {
  console.log(baseUrl);
  const openai = new OpenAI({
    base_url: baseUrl,
    api_key: apiKey,
    default_headers: {
      "astra-api-token": process.env.ASTRA_DB_APPLICATION_TOKEN,
    },
  });

  const stream = await openai.chat.completions.create({
    model: model,
    messages: [{ role: 'user', content: "Can you generate a list of 10 ice cream flavors?" }],
    stream: true,
  });
  for await (const chunk of stream) {
    process.stdout.write(chunk.choices[0]?.delta?.content || '');
  }
}

main(apiKey, model);

[phact]

Hmm seems like you're still hitting openais endpoint?

Since this is js you'll have to do some things manually that we usually do for you in the python sdk wrapper / patch.

I'll take a look at your provided example as soon as I get a chance, don't see anything obviously wrong though.

In the meantime, this might help if you don't mind code diving:

https://github.com/datastax/astra-assistants-api/blob/main/client/astra_assistants/patch.py#L580

[johnpg82]

@phact So the var to change the url in openai is actually baseURL or the .env var OPENAI_BASE_URL. It seems like your example was going to OpenAI the entire time. But when I update the URL I'm getting a new error. I'm going to try just using the CURL method next. I'll post my update here.

BadRequestError: 400 status code (no body) at APIError.generate (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/error.mjs:41:20) at OpenAI.makeStatusError (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:286:25) at OpenAI.makeRequest (file:///Users/jgarland/Documents/GitHub/ai-api/node_modules/openai/core.mjs:330:30) at process.processTicksAndRejections (node:internal/process/task_queues:105:5) at async main (file:///Users/jgarland/Documents/GitHub/ai-api/src/example.js:68:18) { status: 400, headers: { connection: 'keep-alive', 'content-length': '287', 'content-type': 'application/json', date: 'Thu, 14 Nov 2024 22:06:23 GMT' }, request_id: undefined, error: undefined, code: undefined, param: undefined, type: undefined }

[phact]

Thanks! It's entirely possible that the js example may be wrong or outdated.

Maybe sticking a breakpoint inside the SDK might give us some clues. I may be able to have a go at it later tonight.

[johnpg82]

@phact I got it working via Curl and yes the baseUrl just needs updated in the example. Do you have an array of the supported models?

Here is some JS to test all of the different models if it helps. I'd love to get a list of current models so I can know what models I can test with.

import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';

// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });

// Define constants and environment variables
const baseUrl = process.env.OPENAI_BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';
const astraApiToken = process.env.ASTRA_DB_APPLICATION_TOKEN;

// Define model keys and API mappings
const modelApiKeys = {
  'openai': {
    apiKey: process.env.OPENAI_API_KEY,
    models: ['gpt-4o', 'gpt-4o-mini', 'gpt-4']
  },
  'anthropic': {
    apiKey: process.env.ANTHROPIC_API_KEY,
    models: ['anthropic.claude-v2']
  },
  'groq': {
    apiKey: process.env.GROQ_API_KEY,
    models: ['groq/llama3-8b-8192']
  },
  'gemini': {
    apiKey: process.env.GEMINI_API_KEY,
    models: ['gemini-model-1', 'gemini-model-2']
  },
  'perplexity': {
    apiKey: process.env.PERPLEXITYAI_API_KEY,
    models: ['perplexity/llama-3.1-sonar-small-128k-online', 'perplexity/llama-3.1-sonar-huge-128k-online']
  },
  'cohere': {
    apiKey: process.env.COHERE_API_KEY,
    models: ['cohere-model-1', 'cohere-model-2']
  }
};

// Test prompt to use for all models
const testPrompt = "What is 2+2? Reply with just the number.";

async function testModel(model) {
  // Find the API key for this model
  let apiKey;
  for (const { apiKey: key, models } of Object.values(modelApiKeys)) {
    if (models.includes(model)) {
      apiKey = key;
      break;
    }
  }

  if (!apiKey) {
    return { model, error: "API key not found" };
  }

  try {
    const response = await fetch(`${baseUrl}/chat/completions`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'astra-api-token': astraApiToken,
        'Authorization': `Bearer ${apiKey}`,
        'OpenAI-Beta': 'assistants=v2'
      },
      body: JSON.stringify({
        model: model,
        messages: [
          { role: 'system', content: 'You are a helpful assistant.' },
          { role: 'user', content: testPrompt }
        ]
      })
    });

    const data = await response.json();

    if (!response.ok) {
      return {
        model,
        status: response.status,
        error: data
      };
    }

    return {
      model,
      status: response.status,
      message: data.choices[0].message.content
    };
  } catch (error) {
    return {
      model,
      error: error.message
    };
  }
}

async function testAllModels() {
  console.log('Testing all models with prompt:', testPrompt);
  console.log('-----------------------------------');

  // Get all unique models
  const allModels = Object.values(modelApiKeys)
    .flatMap(provider => provider.models);

  // Test each model
  const results = await Promise.all(allModels.map(testModel));

  // Display results in a formatted way
  results.forEach(result => {
    console.log(`\nModel: ${result.model}`);
    if (result.error) {
      console.log('Error:', result.error);
    } else {
      console.log('Status:', result.status);
      console.log('Response:', result.message);
    }
    console.log('-----------------------------------');
  });
}

// Run the tests
testAllModels().catch(error => {
  console.error('Test execution failed:', error);
}); 

[phact]

Everything supported by litellm, I've been pulling them from here:

https://github.com/langflow-ai/langflow/blob/7c048650e00208882fa58f2dee7e32c7c7883de5/src/backend/base/langflow/base/astra_assistants/util.py#L30

[johnpg82]

Thank You! FYI. It looks like the latest version of openai's node library doesn't support overriding the headers.

https://github.com/openai/openai-node/blob/master/src/index.ts

For now I'll use this example to stream.

https://github.com/openai/openai-node/blob/master/examples/stream-to-client-browser.ts

[johnpg82]

Here's' some sample code that will test each of the models.

import * as dotenv from 'dotenv';
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import fetch from 'node-fetch';

// Configure dotenv with the correct path
const __dirname = dirname(fileURLToPath(import.meta.url));
dotenv.config({ path: join(__dirname, '../.env') });

const LITELLM_MODELS_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/refs/heads/main/model_prices_and_context_window.json';
const baseUrl = process.env.BASE_URL || 'https://open-assistant-ai.astra.datastax.com/v1';

// Define API key mapping
const providerApiKeys = {
  'openai': process.env.OPENAI_API_KEY,
  'anthropic': process.env.ANTHROPIC_API_KEY,
  'groq': process.env.GROQ_API_KEY,
  'gemini': process.env.GEMINI_API_KEY,
  'perplexity': process.env.PERPLEXITY_API_KEY,
  'cohere': process.env.COHERE_API_KEY,
};

async function fetchModelData() {
    try {
        const response = await fetch(LITELLM_MODELS_URL);
        const data = await response.json();
        return data;
    } catch (error) {
        console.error('Error fetching model data:', error);
        throw error;
    }
}

async function testModel(model, modelData) {
    const modelInfo = modelData[model];
    if (!modelInfo) return null;

    const provider = modelInfo.litellm_provider;
    const apiKey = providerApiKeys[provider];
    if (!apiKey) return null;

    try {
        const response = await fetch(`${baseUrl}/chat/completions`, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'astra-api-token': process.env.ASTRA_DB_APPLICATION_TOKEN,
                'Authorization': `Bearer ${apiKey}`
            },
            body: JSON.stringify({
                model: model,
                messages: [
                    { role: 'system', content: 'You are a helpful assistant.' },
                    { role: 'user', content: 'What is 2+2? Reply with just the number.' }
                ]
            })
        });

        const data = await response.json();

        if (!response.ok) return null;

        return {
            model,
            provider,
            maxTokens: modelInfo.max_tokens,
            inputCost: modelInfo.input_cost_per_token,
            outputCost: modelInfo.output_cost_per_token
        };

    } catch (error) {
        return null;
    }
}

async function main() {
    try {
        console.log('Testing models...');
        const modelData = await fetchModelData();
        
        // Filter for chat models with matching providers
        const eligibleModels = Object.entries(modelData)
            .filter(([_, value]) => 
                value.mode === 'chat' && 
                value.litellm_provider && 
                providerApiKeys[value.litellm_provider]
            )
            .map(([key]) => key);

        // Test all models and collect working ones
        const results = await Promise.all(
            eligibleModels.map(model => testModel(model, modelData))
        );

        // Filter out null results and format for output
        const workingModels = results
            .filter(result => result !== null)
            .reduce((acc, model) => {
                acc[model.provider] = acc[model.provider] || { 
                    apiKey: `process.env.${model.provider.toUpperCase()}_API_KEY`,
                    models: []
                };
                acc[model.provider].models.push(model.model);
                return acc;
            }, {});

        // Output in a format ready to paste into code
        console.log('\nWorking models configuration:');
        console.log('const modelApiKeys = ' + 
            JSON.stringify(workingModels, null, 2)
            .replace(/"process\.env\./g, 'process.env.')
            .replace(/"/g, "'")
        );

    } catch (error) {
        console.error('Execution error:', error);
    }
}

main().catch(console.error);

[phact]

Thanks!

Hmm, I wonder if providing a custom fetch is the way to mess with headers. https://github.com/openai/openai-node/tree/master?tab=readme-ov-file#customizing-the-fetch-client

[phact]

You motivated me to make a new astra-assistants patch library for node so that you can use the full SDK seamlessly.

Check out this new example:

https://github.com/datastax/astra-assistants-api/blob/main/examples/node/completion/basic-with-patch.mjs

[phact]

Checking in, are you all set?

[johnpg82]

Thanks for checking in. I'm getting started on this now. I'll report back if there are any issues on this thread. I was building the Vector Store aspect on OpenAI First.

[dansoy]

@phact I have tried the patch library for node and I'm getting undefined on all process.env variables. This happens because we are running it on Cloudflare Workers and the env variables are being accessed differently. Can we have the env variables pass as a parameter instead so we can use it on Cloudflare Workers.

[johnpg82]

Hey @dansoy , It looks like you could probably just modify the patch.js

Here's some sample code that you can use.

// env-loader.js

export async function loadEnv(env) {
  globalThis.process = globalThis.process || {};
  process.env = {
    ...(process.env || {}),
    ...env,
  };

  // Optional: allow dotenv in local dev (Miniflare or Node.js)
  if (typeof process?.versions?.node !== 'undefined') {
    try {
      const dotenv = await import('dotenv');
      dotenv.config();
    } catch (e) {
      console.warn('dotenv not loaded:', e.message);
    }
  }
}

Load the Astra-Assistants after you've received your env from the export default fetch.

// worker.js
import { loadEnv } from './env-loader.js';

export default {
  async fetch(request, env, ctx) {
    await loadEnv(env); // Populates process.env for all SDKs

    const { AssistantClient } = await import('astra-assistants'); // Now safe to import

    const client = new AssistantClient({
      baseUrl: process.env.BASE_URL, // or use default
      apiToken: process.env.ASTRA_DB_APPLICATION_TOKEN,
    });

    const response = await client.someMethod({
      // your params here
    });

    return new Response(JSON.stringify(response), {
      headers: { 'content-type': 'application/json' },
    });
  }
};

You may also need to remove dotenv from the patch.js code as well.

https://www.npmjs.com/package/astra-assistants?activeTab=code