Comprehensive Bedrock fix: inference profiles + maxTokens clamping
- ALL models that need inference profiles now use us. prefix IDs: Anthropic (all 8), Nova (3), Meta Llama (3), DeepSeek R1, Writer - Models with low max output tokens (Cohere 4096, Jamba 4096) now have maxOut field; callBedrock clamps maxTokens automatically - Removed Qwen3 235B (no us-east-1 support, no inference profile) - Added better debug logging for response block structure
This commit is contained in:
Daniel Onyejesi
parent
a43468fd4e
commit
208ce27fa4
2 changed files with 41 additions and 31 deletions
|
|
@ -5,7 +5,7 @@
|
|||
// ============================================================
|
||||
|
||||
const { OpenAI } = require('openai');
|
||||
const { DEFAULT_MODEL, FALLBACK_MODEL, getBedrockModelId } = require('./models');
|
||||
const { DEFAULT_MODEL, FALLBACK_MODEL, getBedrockModelId, getBedrockMaxOut } = require('./models');
|
||||
const logger = require('./logger');
|
||||
|
||||
var activeProvider = process.env.AI_PROVIDER || 'openrouter';
|
||||
|
|
@ -158,6 +158,10 @@ async function callBedrock(messages, model, temperature, maxTokens) {
|
|||
var modelId = getBedrockModelId(model);
|
||||
var isAnthropic = modelId.indexOf('anthropic.') !== -1;
|
||||
|
||||
// Clamp maxTokens to model limit if set
|
||||
var modelMax = getBedrockMaxOut(model);
|
||||
if (modelMax && maxTokens > modelMax) maxTokens = modelMax;
|
||||
|
||||
// Separate system message from chat messages
|
||||
var systemMsg = '';
|
||||
var chatMessages = [];
|
||||
|
|
@ -192,7 +196,7 @@ async function callBedrock(messages, model, temperature, maxTokens) {
|
|||
|
||||
// Debug: log response structure for troubleshooting
|
||||
if (Array.isArray(responseBody.content)) {
|
||||
console.log('[Bedrock] Model:', modelId, '| Blocks:', responseBody.content.map(function(b) { return b.type; }).join(', '), '| stop_reason:', responseBody.stop_reason);
|
||||
console.log('[Bedrock] Model:', modelId, '| Blocks:', responseBody.content.length, '|', responseBody.content.map(function(b) { return b.type + '(' + (b.text ? b.text.length : 0) + ')'; }).join(', '), '| stop_reason:', responseBody.stop_reason);
|
||||
}
|
||||
|
||||
// Opus 4.6+ may return multiple content blocks (thinking + text).
|
||||
|
|
|
|||
|
|
@ -27,15 +27,17 @@ var OPENROUTER_MODELS = [
|
|||
];
|
||||
|
||||
var BEDROCK_MODELS = [
|
||||
// ── Anthropic vendor model (verified from AWS docs 2026-03) ──
|
||||
// Models marked "profile" require cross-region inference profiles (us. prefix)
|
||||
// Models marked "on-demand" support direct single-region invocation
|
||||
// All IDs verified against AWS docs 2026-03-24
|
||||
// us. prefix = cross-region inference profile (required for most newer models)
|
||||
// maxOut = model's max output token limit (omit if >= 8192)
|
||||
|
||||
// ── Anthropic vendor model (all use inference profiles) ──
|
||||
{ id: 'anthropic/vendor-model-opus-4-6', name: 'vendor model Opus 4.6', cost: '~$0.075', tag: 'BEST', category: 'premium',
|
||||
bedrockId: 'us.anthropic.agent-config-opus-4-6-v1' },
|
||||
{ id: 'anthropic/vendor-model-opus-4-5', name: 'vendor model Opus 4.5', cost: '~$0.075', tag: 'OPUS', category: 'premium',
|
||||
bedrockId: 'us.anthropic.agent-config-opus-4-5-20251101-v1:0' },
|
||||
{ id: 'anthropic/vendor-model-opus-4-1', name: 'vendor model Opus 4.1', cost: '~$0.075', tag: 'MEDICAL', category: 'premium',
|
||||
bedrockId: 'anthropic.agent-config-opus-4-1-20250805-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
bedrockId: 'us.anthropic.agent-config-opus-4-1-20250805-v1:0' },
|
||||
{ id: 'anthropic/vendor-model-sonnet-4-6', name: 'vendor model Sonnet 4.6', cost: '~$0.015', tag: 'RECOMMENDED', category: 'premium',
|
||||
bedrockId: 'us.anthropic.agent-config-sonnet-4-6' },
|
||||
{ id: 'anthropic/vendor-model-sonnet-4-5', name: 'vendor model Sonnet 4.5', cost: '~$0.015', tag: 'SMART', category: 'premium',
|
||||
|
|
@ -47,53 +49,51 @@ var BEDROCK_MODELS = [
|
|||
{ id: 'anthropic/vendor-model-3.5-haiku', name: 'vendor model 3.5 Haiku', cost: '~$0.002', tag: 'VALUE', category: 'smart',
|
||||
bedrockId: 'us.anthropic.agent-config-3-5-haiku-20241022-v1:0' },
|
||||
|
||||
// ── Amazon Nova (on-demand, fast + cheap) ──
|
||||
// ── Amazon Nova (inference profiles) ──
|
||||
{ id: 'amazon/nova-pro', name: 'Amazon Nova Pro', cost: '~$0.008', tag: 'SMART', category: 'smart',
|
||||
bedrockId: 'amazon.nova-pro-v1:0', regions: ['us-east-1', 'ap-southeast-2', 'ap-southeast-3', 'eu-west-2', 'me-central-1'] },
|
||||
bedrockId: 'us.amazon.nova-pro-v1:0' },
|
||||
{ id: 'amazon/nova-lite', name: 'Amazon Nova Lite', cost: '~$0.001', tag: 'FAST', category: 'fast',
|
||||
bedrockId: 'amazon.nova-lite-v1:0', regions: ['us-east-1', 'ap-northeast-1', 'ap-southeast-2', 'ap-southeast-3', 'eu-north-1', 'eu-west-2', 'me-central-1'] },
|
||||
bedrockId: 'us.amazon.nova-lite-v1:0' },
|
||||
{ id: 'amazon/nova-micro', name: 'Amazon Nova Micro', cost: '~$0.0004', tag: 'CHEAPEST', category: 'fast',
|
||||
bedrockId: 'amazon.nova-micro-v1:0', regions: ['us-east-1', 'ap-southeast-2', 'eu-west-2'] },
|
||||
bedrockId: 'us.amazon.nova-micro-v1:0' },
|
||||
|
||||
// ── Meta Llama (on-demand, good for clinical summarisation) ──
|
||||
// ── Meta Llama (inference profiles) ──
|
||||
{ id: 'meta/llama-4-maverick', name: 'Llama 4 Maverick', cost: '~$0.005', tag: 'NEW', category: 'smart',
|
||||
bedrockId: 'meta.llama4-maverick-17b-instruct-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] },
|
||||
bedrockId: 'us.meta.llama4-maverick-17b-instruct-v1:0' },
|
||||
{ id: 'meta/llama-4-scout', name: 'Llama 4 Scout', cost: '~$0.004', tag: 'NEW', category: 'smart',
|
||||
bedrockId: 'meta.llama4-scout-17b-instruct-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] },
|
||||
bedrockId: 'us.meta.llama4-scout-17b-instruct-v1:0' },
|
||||
{ id: 'meta/llama-3.3-70b', name: 'Llama 3.3 70B', cost: '~$0.003', tag: 'GOOD', category: 'smart',
|
||||
bedrockId: 'meta.llama3-3-70b-instruct-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
bedrockId: 'us.meta.llama3-3-70b-instruct-v1:0' },
|
||||
|
||||
// ── DeepSeek (reasoning + fast) ──
|
||||
// ── DeepSeek (R1 has profile, V3.2 is on-demand) ──
|
||||
{ id: 'deepseek/r1', name: 'DeepSeek R1', cost: '~$0.005', tag: 'REASONING', category: 'smart',
|
||||
bedrockId: 'deepseek.r1-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
bedrockId: 'us.deepseek.r1-v1:0' },
|
||||
{ id: 'deepseek/v3.2', name: 'DeepSeek V3.2', cost: '~$0.001', tag: 'FAST', category: 'fast',
|
||||
bedrockId: 'deepseek.v3.2', regions: ['us-east-1', 'us-east-2', 'us-west-2', 'ap-northeast-1', 'eu-north-1'] },
|
||||
bedrockId: 'deepseek.v3.2', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
|
||||
// ── Mistral AI ──
|
||||
// ── Mistral AI (on-demand, no profiles available) ──
|
||||
{ id: 'mistral/large-3', name: 'Mistral Large 3 (675B)', cost: '~$0.008', tag: 'PREMIUM', category: 'premium',
|
||||
bedrockId: 'mistral.mistral-large-3-675b-instruct', regions: ['us-east-1', 'us-east-2', 'us-west-2', 'ap-northeast-1'] },
|
||||
bedrockId: 'mistral.mistral-large-3-675b-instruct', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
{ id: 'mistral/magistral-small', name: 'Magistral Small', cost: '~$0.003', tag: 'VALUE', category: 'smart',
|
||||
bedrockId: 'mistral.magistral-small-2509', regions: ['us-east-1', 'us-east-2', 'us-west-2', 'ap-northeast-1'] },
|
||||
bedrockId: 'mistral.magistral-small-2509', regions: ['us-east-1', 'us-east-2', 'us-west-2'] },
|
||||
|
||||
// ── Cohere (good for medical text, us-east-1/us-west-2 only) ──
|
||||
// ── Cohere (on-demand, max 4096 output) ──
|
||||
{ id: 'cohere/command-r-plus', name: 'Command R+', cost: '~$0.005', tag: 'GOOD', category: 'smart',
|
||||
bedrockId: 'cohere.command-r-plus-v1:0', regions: ['us-east-1', 'us-west-2'] },
|
||||
bedrockId: 'cohere.command-r-plus-v1:0', regions: ['us-east-1', 'us-west-2'], maxOut: 4096 },
|
||||
{ id: 'cohere/command-r', name: 'Command R', cost: '~$0.002', tag: 'VALUE', category: 'fast',
|
||||
bedrockId: 'cohere.command-r-v1:0', regions: ['us-east-1', 'us-west-2'] },
|
||||
bedrockId: 'cohere.command-r-v1:0', regions: ['us-east-1', 'us-west-2'], maxOut: 4096 },
|
||||
|
||||
// ── AI21 Labs (us-east-1 only) ──
|
||||
// ── AI21 Labs (on-demand, max 4096 output) ──
|
||||
{ id: 'ai21/jamba-1.5-large', name: 'Jamba 1.5 Large', cost: '~$0.005', tag: 'GOOD', category: 'smart',
|
||||
bedrockId: 'ai21.jamba-1-5-large-v1:0', regions: ['us-east-1'] },
|
||||
bedrockId: 'ai21.jamba-1-5-large-v1:0', regions: ['us-east-1'], maxOut: 4096 },
|
||||
|
||||
// ── Writer (medical/clinical writing, us regions) ──
|
||||
// ── Writer (inference profile) ──
|
||||
{ id: 'writer/palmyra-x5', name: 'Palmyra X5', cost: '~$0.01', tag: 'CLINICAL', category: 'premium',
|
||||
bedrockId: 'writer.palmyra-x5-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] },
|
||||
bedrockId: 'us.writer.palmyra-x5-v1:0' },
|
||||
|
||||
// ── Qwen (strong multilingual medical reasoning) ──
|
||||
{ id: 'qwen/qwen3-235b', name: 'Qwen3 235B', cost: '~$0.005', tag: 'SMART', category: 'smart',
|
||||
bedrockId: 'qwen.qwen3-235b-a22b-2507-v1:0', regions: ['us-east-2', 'us-west-2', 'ap-northeast-1', 'eu-central-1'] },
|
||||
// ── Qwen (on-demand, no profiles) ──
|
||||
{ id: 'qwen/qwen3-32b', name: 'Qwen3 32B', cost: '~$0.002', tag: 'VALUE', category: 'fast',
|
||||
bedrockId: 'qwen.qwen3-32b-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-2', 'ap-northeast-1', 'eu-central-1'] }
|
||||
bedrockId: 'qwen.qwen3-32b-v1:0', regions: ['us-east-1', 'us-east-2', 'us-west-2'] }
|
||||
];
|
||||
|
||||
var AZURE_MODELS = [
|
||||
|
|
@ -143,6 +143,11 @@ function getBedrockModelId(modelId) {
|
|||
return found ? found.bedrockId : modelId;
|
||||
}
|
||||
|
||||
function getBedrockMaxOut(modelId) {
|
||||
var found = BEDROCK_MODELS.find(function(m) { return m.id === modelId; });
|
||||
return found && found.maxOut ? found.maxOut : null;
|
||||
}
|
||||
|
||||
var AVAILABLE_MODELS = getAvailableModels();
|
||||
var DEFAULT_MODEL = getDefaultModel();
|
||||
var FALLBACK_MODEL = getFallbackModel();
|
||||
|
|
@ -184,5 +189,6 @@ module.exports = {
|
|||
getDefaultModel,
|
||||
getFallbackModel,
|
||||
getBedrockModelId,
|
||||
getBedrockMaxOut,
|
||||
activeProvider
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue