refactor(compute): remove legacy app whisper duplicate and unify core whisper boundary
This commit is contained in:
Richard R
parent
1612e6694b
commit
37d11bf9b8
18 changed files with 29 additions and 1887 deletions
|
|
@ -65,7 +65,7 @@ COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
|||
RUN chmod +x /usr/local/bin/weed
|
||||
|
||||
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
|
||||
COPY --from=app-builder /app/src/lib/server/whisper/model/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||
COPY --from=app-builder /app/compute/core/src/whisper/model/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||
|
||||
# Expose the port the app runs on
|
||||
EXPOSE 3003
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
"./contracts": "./src/contracts.ts",
|
||||
"./local-runtime": "./src/local-runtime.ts",
|
||||
"./runtime": "./src/runtime/index.ts",
|
||||
"./whisper": "./src/whisper/index.ts",
|
||||
"./pdf-layout": "./src/pdf-layout/index.ts",
|
||||
"./pdf-layout/parse": "./src/pdf-layout/parsePdf.ts"
|
||||
}
|
||||
|
|
|
|||
18
compute/core/src/whisper/index.ts
Normal file
18
compute/core/src/whisper/index.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
export {
|
||||
alignAudioWithText,
|
||||
makeWhisperCacheKey,
|
||||
type WhisperRequestBody,
|
||||
} from './alignment';
|
||||
|
||||
export {
|
||||
ensureWhisperModel,
|
||||
ensureWhisperArtifacts,
|
||||
createSingleflightRunner,
|
||||
type WhisperArtifactSpec,
|
||||
type WhisperStaticArtifactSpec,
|
||||
type WhisperFetch,
|
||||
} from './ensureModel';
|
||||
|
||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './alignment-mapping';
|
||||
export { buildGoertzelCoefficients, goertzelPower } from './spectral';
|
||||
export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './token-timestamps';
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import os from 'node:os';
|
||||
import Fastify, { type FastifyRequest } from 'fastify';
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
|
|
@ -29,6 +28,8 @@ import {
|
|||
} from '@openreader/compute-core/local-runtime';
|
||||
import {
|
||||
getComputeTimeoutConfig,
|
||||
getAvailableCpuCores,
|
||||
getOnnxThreadsPerJob,
|
||||
withIdleTimeoutAndHardCap,
|
||||
withTimeout,
|
||||
} from '@openreader/compute-core/runtime';
|
||||
|
|
@ -117,20 +118,6 @@ function parseBoolEnv(name: string, fallback: boolean): boolean {
|
|||
return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on';
|
||||
}
|
||||
|
||||
function getAvailableCpuCores(): number {
|
||||
if (typeof os.availableParallelism === 'function') {
|
||||
const value = os.availableParallelism();
|
||||
if (Number.isFinite(value) && value >= 1) return Math.floor(value);
|
||||
}
|
||||
const fallback = os.cpus().length;
|
||||
return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1;
|
||||
}
|
||||
|
||||
function getOnnxThreadsPerJob(jobConcurrency: number): number {
|
||||
const usableCores = Math.max(1, getAvailableCpuCores() - 1);
|
||||
return Math.max(1, Math.floor(usableCores / Math.max(1, jobConcurrency)));
|
||||
}
|
||||
|
||||
function buildLoggerConfig(): boolean | Record<string, unknown> {
|
||||
const format = (process.env.COMPUTE_LOG_FORMAT?.trim().toLowerCase() || 'pretty');
|
||||
if (format === 'json') return true;
|
||||
|
|
@ -528,7 +515,7 @@ async function main(): Promise<void> {
|
|||
pdfAttempts,
|
||||
opStaleMs,
|
||||
availableCpuCores: getAvailableCpuCores(),
|
||||
onnxThreadsPerJob: getOnnxThreadsPerJob(jobConcurrency),
|
||||
onnxThreadsPerJob: getOnnxThreadsPerJob(),
|
||||
natsApiTimeoutMs: NATS_API_TIMEOUT_MS,
|
||||
pdfLayoutHardCapMs: pdfHardCapMs,
|
||||
}, 'compute runtime config');
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts';
|
||||
import { preprocessSentenceForAudio } from '@/lib/shared/nlp';
|
||||
|
||||
export interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||
const normalizedSentence = preprocessSentenceForAudio(sentence);
|
||||
const lowerSentence = normalizedSentence.toLowerCase();
|
||||
let cursor = 0;
|
||||
|
||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
||||
const token = w.word.trim();
|
||||
if (!token) {
|
||||
return {
|
||||
text: '',
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: cursor,
|
||||
charEnd: cursor,
|
||||
};
|
||||
}
|
||||
|
||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||
const start = idx >= 0 ? idx : cursor;
|
||||
const end = Math.min(normalizedSentence.length, start + token.length);
|
||||
cursor = Math.max(cursor, end);
|
||||
|
||||
return {
|
||||
text: token,
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: start,
|
||||
charEnd: end,
|
||||
};
|
||||
}).filter((word) => word.text.length > 0);
|
||||
|
||||
return {
|
||||
sentence,
|
||||
sentenceIndex: 0,
|
||||
words: alignedWords,
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,240 +0,0 @@
|
|||
import path from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises';
|
||||
import { DOCSTORE_DIR } from '@/lib/server/storage/library-mount';
|
||||
import manifest from '@/lib/server/whisper/model/manifest.json';
|
||||
|
||||
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped');
|
||||
const STATIC_LICENSE_PATH = path.join(process.cwd(), 'src/lib/server/whisper/model/LICENSE.txt');
|
||||
|
||||
export const WHISPER_CONFIG_PATH = path.join(MODEL_DIR, 'config.json');
|
||||
export const WHISPER_GENERATION_CONFIG_PATH = path.join(MODEL_DIR, 'generation_config.json');
|
||||
export const WHISPER_TOKENIZER_PATH = path.join(MODEL_DIR, 'tokenizer.json');
|
||||
export const WHISPER_TOKENIZER_CONFIG_PATH = path.join(MODEL_DIR, 'tokenizer_config.json');
|
||||
export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_int8.onnx');
|
||||
export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_int8.onnx');
|
||||
export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_int8.onnx');
|
||||
|
||||
const BASE_MODEL_URL = 'https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main';
|
||||
const WHISPER_MODEL_BASE_URL_ENV = 'WHISPER_MODEL_BASE_URL';
|
||||
|
||||
const MODEL_RELATIVE_PATHS: string[] = [
|
||||
'config.json',
|
||||
'generation_config.json',
|
||||
'tokenizer.json',
|
||||
'tokenizer_config.json',
|
||||
'merges.txt',
|
||||
'vocab.json',
|
||||
'normalizer.json',
|
||||
'added_tokens.json',
|
||||
'preprocessor_config.json',
|
||||
'special_tokens_map.json',
|
||||
'onnx/encoder_model_int8.onnx',
|
||||
'onnx/decoder_model_merged_int8.onnx',
|
||||
'onnx/decoder_with_past_model_int8.onnx',
|
||||
];
|
||||
|
||||
const DEFAULT_URLS: Record<string, string> = {
|
||||
'config.json': `${BASE_MODEL_URL}/config.json`,
|
||||
'generation_config.json': `${BASE_MODEL_URL}/generation_config.json`,
|
||||
'tokenizer.json': `${BASE_MODEL_URL}/tokenizer.json`,
|
||||
'tokenizer_config.json': `${BASE_MODEL_URL}/tokenizer_config.json`,
|
||||
'merges.txt': `${BASE_MODEL_URL}/merges.txt`,
|
||||
'vocab.json': `${BASE_MODEL_URL}/vocab.json`,
|
||||
'normalizer.json': `${BASE_MODEL_URL}/normalizer.json`,
|
||||
'added_tokens.json': `${BASE_MODEL_URL}/added_tokens.json`,
|
||||
'preprocessor_config.json': `${BASE_MODEL_URL}/preprocessor_config.json`,
|
||||
'special_tokens_map.json': `${BASE_MODEL_URL}/special_tokens_map.json`,
|
||||
'onnx/encoder_model_int8.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_int8.onnx`,
|
||||
'onnx/decoder_model_merged_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_int8.onnx`,
|
||||
'onnx/decoder_with_past_model_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_int8.onnx`,
|
||||
};
|
||||
|
||||
type ManifestEntry = { path: string; sha256?: string; size?: number };
|
||||
|
||||
export interface WhisperArtifactSpec {
|
||||
path: string;
|
||||
sha256?: string;
|
||||
size?: number;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface WhisperStaticArtifactSpec {
|
||||
path: string;
|
||||
sha256?: string;
|
||||
size?: number;
|
||||
sourcePath: string;
|
||||
}
|
||||
|
||||
export type WhisperFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
||||
|
||||
const MANIFEST_FILES = manifest.files as ManifestEntry[];
|
||||
const MODEL_FILES = MANIFEST_FILES.filter((entry) => entry.path !== 'LICENSE.txt');
|
||||
const LICENSE_FILE = MANIFEST_FILES.find((entry) => entry.path === 'LICENSE.txt');
|
||||
|
||||
function normalizeExpected(entry: { sha256?: string; size?: number }): { sha256: string | null; size: number } {
|
||||
return {
|
||||
sha256: typeof entry.sha256 === 'string' ? entry.sha256.toLowerCase() : null,
|
||||
size: Number(entry.size ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
function resolvePath(relativePath: string, modelDir: string): string {
|
||||
return path.join(modelDir, relativePath);
|
||||
}
|
||||
|
||||
function joinModelUrl(baseUrl: string, relativePath: string): string {
|
||||
return `${baseUrl.replace(/\/+$/, '')}/${relativePath}`;
|
||||
}
|
||||
|
||||
function resolveUrl(relativePath: string): string {
|
||||
const overrideBase = process.env[WHISPER_MODEL_BASE_URL_ENV]?.trim();
|
||||
if (overrideBase) {
|
||||
return joinModelUrl(overrideBase, relativePath);
|
||||
}
|
||||
const fallback = DEFAULT_URLS[relativePath];
|
||||
if (!fallback) {
|
||||
throw new Error(`No default URL configured for Whisper model artifact: ${relativePath}`);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function sha256OfBytes(bytes: Uint8Array): string {
|
||||
return createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
|
||||
function verifyBytes(bytes: Uint8Array, expected: { sha256?: string; size?: number }): boolean {
|
||||
const normalized = normalizeExpected(expected);
|
||||
if (Number.isFinite(normalized.size) && normalized.size > 0 && bytes.byteLength !== normalized.size) {
|
||||
return false;
|
||||
}
|
||||
if (!normalized.sha256) return true;
|
||||
return sha256OfBytes(bytes) === normalized.sha256;
|
||||
}
|
||||
|
||||
async function verifyFile(filePath: string, expected: { sha256?: string; size?: number }): Promise<boolean> {
|
||||
const bytes = await readFile(filePath);
|
||||
return verifyBytes(bytes, expected);
|
||||
}
|
||||
|
||||
async function downloadToFile(fetchImpl: WhisperFetch, url: string, outPath: string): Promise<void> {
|
||||
const res = await fetchImpl(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Download failed for ${url}: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
await writeFile(outPath, bytes);
|
||||
}
|
||||
|
||||
export async function ensureWhisperArtifacts(options: {
|
||||
modelDir: string;
|
||||
artifacts: WhisperArtifactSpec[];
|
||||
staticArtifacts?: WhisperStaticArtifactSpec[];
|
||||
fetchImpl?: WhisperFetch;
|
||||
}): Promise<void> {
|
||||
const {
|
||||
modelDir,
|
||||
artifacts,
|
||||
staticArtifacts = [],
|
||||
fetchImpl = fetch,
|
||||
} = options;
|
||||
|
||||
try {
|
||||
await Promise.all(artifacts.map(async (artifact) => {
|
||||
const target = resolvePath(artifact.path, modelDir);
|
||||
await access(target);
|
||||
const valid = await verifyFile(target, artifact);
|
||||
if (!valid) {
|
||||
throw new Error(`Checksum mismatch for existing Whisper artifact: ${artifact.path}`);
|
||||
}
|
||||
}));
|
||||
|
||||
await Promise.all(staticArtifacts.map(async (artifact) => {
|
||||
const target = resolvePath(artifact.path, modelDir);
|
||||
await access(target);
|
||||
const valid = await verifyFile(target, artifact);
|
||||
if (!valid) {
|
||||
throw new Error(`Checksum mismatch for existing Whisper static artifact: ${artifact.path}`);
|
||||
}
|
||||
}));
|
||||
|
||||
return;
|
||||
} catch {
|
||||
// Continue to repair/download.
|
||||
}
|
||||
|
||||
for (const artifact of artifacts) {
|
||||
const target = resolvePath(artifact.path, modelDir);
|
||||
const targetDir = path.dirname(target);
|
||||
const tmp = `${target}.tmp`;
|
||||
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
await downloadToFile(fetchImpl, artifact.url, tmp);
|
||||
if (!(await verifyFile(tmp, artifact))) {
|
||||
await unlink(tmp).catch(() => undefined);
|
||||
throw new Error(`Whisper artifact checksum verification failed: ${artifact.path}`);
|
||||
}
|
||||
await rename(tmp, target);
|
||||
}
|
||||
|
||||
for (const artifact of staticArtifacts) {
|
||||
const target = resolvePath(artifact.path, modelDir);
|
||||
const targetDir = path.dirname(target);
|
||||
await mkdir(targetDir, { recursive: true });
|
||||
await copyFile(artifact.sourcePath, target);
|
||||
if (!(await verifyFile(target, artifact))) {
|
||||
throw new Error(`Whisper static artifact checksum verification failed: ${artifact.path}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function createSingleflightRunner<T>(work: () => Promise<T>): () => Promise<T> {
|
||||
let inflight: Promise<T> | null = null;
|
||||
return async () => {
|
||||
if (inflight) return inflight;
|
||||
inflight = work().finally(() => {
|
||||
inflight = null;
|
||||
});
|
||||
return inflight;
|
||||
};
|
||||
}
|
||||
|
||||
async function ensureModelInternal(): Promise<string> {
|
||||
if (process.env[WHISPER_MODEL_BASE_URL_ENV]?.trim()) {
|
||||
for (const relativePath of MODEL_RELATIVE_PATHS) {
|
||||
if (!(relativePath in DEFAULT_URLS)) {
|
||||
throw new Error(`Missing default URL path mapping for Whisper artifact: ${relativePath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const artifacts: WhisperArtifactSpec[] = MODEL_FILES.map((entry) => ({
|
||||
path: entry.path,
|
||||
sha256: entry.sha256,
|
||||
size: entry.size,
|
||||
url: resolveUrl(entry.path),
|
||||
}));
|
||||
|
||||
const staticArtifacts: WhisperStaticArtifactSpec[] = LICENSE_FILE
|
||||
? [{
|
||||
path: LICENSE_FILE.path,
|
||||
sha256: LICENSE_FILE.sha256,
|
||||
size: LICENSE_FILE.size,
|
||||
sourcePath: STATIC_LICENSE_PATH,
|
||||
}]
|
||||
: [];
|
||||
|
||||
await ensureWhisperArtifacts({
|
||||
modelDir: MODEL_DIR,
|
||||
artifacts,
|
||||
staticArtifacts,
|
||||
});
|
||||
|
||||
return WHISPER_ENCODER_MODEL_PATH;
|
||||
}
|
||||
|
||||
const ensureWhisperModelSingleflight = createSingleflightRunner(ensureModelInternal);
|
||||
|
||||
export async function ensureWhisperModel(): Promise<string> {
|
||||
return ensureWhisperModelSingleflight();
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2022 OpenAI
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
{
|
||||
"name": "whisper-base_timestamped-int8",
|
||||
"version": "onnx-community/whisper-base_timestamped@608c49e61301901684bc36cac8f74b95ff6b5a8e",
|
||||
"files": [
|
||||
{
|
||||
"path": "config.json",
|
||||
"sha256": "f4d0608f7d918166da7edb3e188de5ef1bfe70d9802e785d271fd88111e9cf4b",
|
||||
"size": 2243
|
||||
},
|
||||
{
|
||||
"path": "generation_config.json",
|
||||
"sha256": "61070cf8de25b1e9256e8e102ded49d8d24a8369ed36ef84fdf21549e68125a0",
|
||||
"size": 3832
|
||||
},
|
||||
{
|
||||
"path": "tokenizer.json",
|
||||
"sha256": "27fc476bfe7f17299480be2273fc0608e4d5a99aba2ab5dec5374b4482d1a566",
|
||||
"size": 2480466
|
||||
},
|
||||
{
|
||||
"path": "tokenizer_config.json",
|
||||
"sha256": "2e036e4dbacfdeb7242c7d4ec4149f4a16e86026048f94d1637e3a8ee9c6a573",
|
||||
"size": 282682
|
||||
},
|
||||
{
|
||||
"path": "merges.txt",
|
||||
"sha256": "2df2990a395e35e8dfbc7511e08c12d56018d8d04691e0133e5d63b21e154dc6",
|
||||
"size": 493869
|
||||
},
|
||||
{
|
||||
"path": "vocab.json",
|
||||
"sha256": "50d6a919f0a0601d56a04eb583c780d18553aa388254ba3158eb6a00f13e2c1a",
|
||||
"size": 1036584
|
||||
},
|
||||
{
|
||||
"path": "normalizer.json",
|
||||
"sha256": "bf1c507dc8724ca9cf9903640dacfb69dae2f00edee4f21ceba106a7392f26dd",
|
||||
"size": 52666
|
||||
},
|
||||
{
|
||||
"path": "added_tokens.json",
|
||||
"sha256": "9715fd2243b6f06a5858b5e32950d2853f73dd5bc201aafcf76f5082a2d8acd1",
|
||||
"size": 34604
|
||||
},
|
||||
{
|
||||
"path": "preprocessor_config.json",
|
||||
"sha256": "a6a76d28c93edb273669eb9e0b0636a2bddbb1272c3261e47b7ca6dfdbac1b8d",
|
||||
"size": 339
|
||||
},
|
||||
{
|
||||
"path": "special_tokens_map.json",
|
||||
"sha256": "e67ae3a0aaa99abcd9f187138e12db1f65c16a14761c50ef10eef2c174a7a691",
|
||||
"size": 2194
|
||||
},
|
||||
{
|
||||
"path": "onnx/encoder_model_int8.onnx",
|
||||
"sha256": "152da96dd8ff3f28f3fadabc2e8960405a277846453ff94ed411fe935a72917f",
|
||||
"size": 23159150
|
||||
},
|
||||
{
|
||||
"path": "onnx/decoder_model_merged_int8.onnx",
|
||||
"sha256": "cf9a8d5bcddc0917a0078135b484cedcaf44f28909cd91910abd29dced9171db",
|
||||
"size": 53712708
|
||||
},
|
||||
{
|
||||
"path": "onnx/decoder_with_past_model_int8.onnx",
|
||||
"sha256": "bdd92860d0ed7dff2aca623963378cbba1b617bfae127356db1c8aa8baa930ef",
|
||||
"size": 50131672
|
||||
},
|
||||
{
|
||||
"path": "LICENSE.txt",
|
||||
"sha256": "b5d65a59060e68c4ff940e1eddfa6f94b2d68fdf58ed7f4dd57721c997e35e9d",
|
||||
"size": 1063
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -1,21 +0,0 @@
|
|||
export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array {
|
||||
const coeffs = new Float64Array(freqBins);
|
||||
for (let k = 0; k < freqBins; k += 1) {
|
||||
coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize);
|
||||
}
|
||||
return coeffs;
|
||||
}
|
||||
|
||||
export function goertzelPower(samples: Float32Array, coeff: number): number {
|
||||
let s1 = 0;
|
||||
let s2 = 0;
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const s0 = samples[i] + (coeff * s1) - s2;
|
||||
s2 = s1;
|
||||
s1 = s0;
|
||||
}
|
||||
|
||||
const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2);
|
||||
if (!Number.isFinite(power) || power < 0) return 0;
|
||||
return power;
|
||||
}
|
||||
|
|
@ -1,449 +0,0 @@
|
|||
import type { Tokenizer } from '@huggingface/tokenizers';
|
||||
import type * as ort from 'onnxruntime-node';
|
||||
|
||||
const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E';
|
||||
const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu');
|
||||
|
||||
type TokenTimestamp = [start: number, end: number];
|
||||
|
||||
export interface WhisperWordTiming {
|
||||
word: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
}
|
||||
|
||||
function medianFilter(data: Float32Array, windowSize: number): Float32Array {
|
||||
if (windowSize % 2 === 0 || windowSize <= 0) {
|
||||
throw new Error('Window size must be a positive odd number');
|
||||
}
|
||||
|
||||
const output = new Float32Array(data.length);
|
||||
const buffer = new Float32Array(windowSize);
|
||||
const halfWindow = Math.floor(windowSize / 2);
|
||||
|
||||
for (let i = 0; i < data.length; i += 1) {
|
||||
let valuesIndex = 0;
|
||||
for (let j = -halfWindow; j <= halfWindow; j += 1) {
|
||||
let index = i + j;
|
||||
if (index < 0) {
|
||||
index = Math.abs(index);
|
||||
} else if (index >= data.length) {
|
||||
index = (2 * (data.length - 1)) - index;
|
||||
}
|
||||
buffer[valuesIndex] = data[index];
|
||||
valuesIndex += 1;
|
||||
}
|
||||
|
||||
const sortable = Array.from(buffer);
|
||||
sortable.sort((a, b) => a - b);
|
||||
output[i] = sortable[halfWindow] ?? 0;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function dynamicTimeWarping(matrix: Float32Array[], rows: number, cols: number): [number[], number[]] {
|
||||
const cost: number[][] = Array.from({ length: rows + 1 }, () => Array(cols + 1).fill(Number.POSITIVE_INFINITY));
|
||||
const trace: number[][] = Array.from({ length: rows + 1 }, () => Array(cols + 1).fill(-1));
|
||||
cost[0][0] = 0;
|
||||
|
||||
for (let j = 1; j <= cols; j += 1) {
|
||||
for (let i = 1; i <= rows; i += 1) {
|
||||
const c0 = cost[i - 1][j - 1];
|
||||
const c1 = cost[i - 1][j];
|
||||
const c2 = cost[i][j - 1];
|
||||
let c: number;
|
||||
let t: number;
|
||||
if (c0 < c1 && c0 < c2) {
|
||||
c = c0;
|
||||
t = 0;
|
||||
} else if (c1 < c0 && c1 < c2) {
|
||||
c = c1;
|
||||
t = 1;
|
||||
} else {
|
||||
c = c2;
|
||||
t = 2;
|
||||
}
|
||||
cost[i][j] = matrix[i - 1][j - 1] + c;
|
||||
trace[i][j] = t;
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i <= cols; i += 1) trace[0][i] = 2;
|
||||
for (let i = 0; i <= rows; i += 1) trace[i][0] = 1;
|
||||
|
||||
let i = rows;
|
||||
let j = cols;
|
||||
const textIndices: number[] = [];
|
||||
const timeIndices: number[] = [];
|
||||
while (i > 0 || j > 0) {
|
||||
textIndices.push(i - 1);
|
||||
timeIndices.push(j - 1);
|
||||
const step = trace[i][j];
|
||||
if (step === 0) {
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if (step === 1) {
|
||||
i -= 1;
|
||||
} else if (step === 2) {
|
||||
j -= 1;
|
||||
} else {
|
||||
throw new Error(`Unexpected DTW trace state at [${i}, ${j}]`);
|
||||
}
|
||||
}
|
||||
|
||||
textIndices.reverse();
|
||||
timeIndices.reverse();
|
||||
return [textIndices, timeIndices];
|
||||
}
|
||||
|
||||
function round2(value: number): number {
|
||||
return Math.round(value * 100) / 100;
|
||||
}
|
||||
|
||||
function decodeTokens(tokenizer: Pick<Tokenizer, 'decode'>, tokens: number[]): string {
|
||||
return tokenizer.decode(tokens, { skip_special_tokens: false });
|
||||
}
|
||||
|
||||
function splitTokensOnUnicode(
|
||||
tokenizer: Pick<Tokenizer, 'decode'>,
|
||||
tokens: number[],
|
||||
): [string[], number[][], number[][]] {
|
||||
const decodedFull = decodeTokens(tokenizer, tokens);
|
||||
const replacementChar = '\uFFFD';
|
||||
const words: string[] = [];
|
||||
const wordTokens: number[][] = [];
|
||||
const tokenIndices: number[][] = [];
|
||||
let currentTokens: number[] = [];
|
||||
let currentIndices: number[] = [];
|
||||
let unicodeOffset = 0;
|
||||
|
||||
for (let i = 0; i < tokens.length; i += 1) {
|
||||
currentTokens.push(tokens[i]);
|
||||
currentIndices.push(i);
|
||||
|
||||
const decoded = decodeTokens(tokenizer, currentTokens);
|
||||
if (
|
||||
!decoded.includes(replacementChar)
|
||||
|| decodedFull[unicodeOffset + decoded.indexOf(replacementChar)] === replacementChar
|
||||
) {
|
||||
words.push(decoded);
|
||||
wordTokens.push(currentTokens);
|
||||
tokenIndices.push(currentIndices);
|
||||
currentTokens = [];
|
||||
currentIndices = [];
|
||||
unicodeOffset += decoded.length;
|
||||
}
|
||||
}
|
||||
|
||||
return [words, wordTokens, tokenIndices];
|
||||
}
|
||||
|
||||
function splitTokensOnSpaces(
|
||||
tokenizer: Pick<Tokenizer, 'decode'>,
|
||||
tokens: number[],
|
||||
eosTokenId: number,
|
||||
): [string[], number[][], number[][]] {
|
||||
const [subwords, subwordTokens, subwordIndices] = splitTokensOnUnicode(tokenizer, tokens);
|
||||
const words: string[] = [];
|
||||
const wordTokens: number[][] = [];
|
||||
const tokenIndices: number[][] = [];
|
||||
|
||||
for (let i = 0; i < subwords.length; i += 1) {
|
||||
const subword = subwords[i];
|
||||
const tokenList = subwordTokens[i];
|
||||
const indices = subwordIndices[i];
|
||||
const special = tokenList[0] >= eosTokenId;
|
||||
const withSpace = subword.startsWith(' ');
|
||||
const trimmed = subword.trim();
|
||||
const punctuation = PUNCTUATION_ONLY_REGEX.test(trimmed);
|
||||
|
||||
if (special || withSpace || punctuation || words.length === 0) {
|
||||
words.push(subword);
|
||||
wordTokens.push([...tokenList]);
|
||||
tokenIndices.push([...indices]);
|
||||
} else {
|
||||
const ix = words.length - 1;
|
||||
words[ix] += subword;
|
||||
wordTokens[ix].push(...tokenList);
|
||||
tokenIndices[ix].push(...indices);
|
||||
}
|
||||
}
|
||||
|
||||
return [words, wordTokens, tokenIndices];
|
||||
}
|
||||
|
||||
function mergePunctuations(
|
||||
words: string[],
|
||||
tokens: number[][],
|
||||
indices: number[][],
|
||||
prependPunctuations = '"\'“¡¿([{-',
|
||||
appendPunctuations = '"\'.。,，!！?？:：”)]}、',
|
||||
): [string[], number[][], number[][]] {
|
||||
const newWords = words.map((w) => `${w}`);
|
||||
const newTokens = tokens.map((t) => [...t]);
|
||||
const newIndices = indices.map((idx) => [...idx]);
|
||||
|
||||
let i = newWords.length - 2;
|
||||
let j = newWords.length - 1;
|
||||
while (i >= 0) {
|
||||
if (newWords[i].startsWith(' ') && prependPunctuations.includes(newWords[i].trim())) {
|
||||
newWords[j] = newWords[i] + newWords[j];
|
||||
newTokens[j] = [...newTokens[i], ...newTokens[j]];
|
||||
newIndices[j] = [...newIndices[i], ...newIndices[j]];
|
||||
newWords[i] = '';
|
||||
newTokens[i] = [];
|
||||
newIndices[i] = [];
|
||||
} else {
|
||||
j = i;
|
||||
}
|
||||
i -= 1;
|
||||
}
|
||||
|
||||
i = 0;
|
||||
j = 1;
|
||||
while (j < newWords.length) {
|
||||
if (!newWords[i].endsWith(' ') && appendPunctuations.includes(newWords[j])) {
|
||||
newWords[i] += newWords[j];
|
||||
newTokens[i] = [...newTokens[i], ...newTokens[j]];
|
||||
newIndices[i] = [...newIndices[i], ...newIndices[j]];
|
||||
newWords[j] = '';
|
||||
newTokens[j] = [];
|
||||
newIndices[j] = [];
|
||||
} else {
|
||||
i = j;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
|
||||
return [
|
||||
newWords.filter((w) => w.length > 0),
|
||||
newTokens.filter((t) => t.length > 0),
|
||||
newIndices.filter((t) => t.length > 0),
|
||||
];
|
||||
}
|
||||
|
||||
function combineTokensIntoWords(
|
||||
tokenizer: Pick<Tokenizer, 'decode'>,
|
||||
tokens: number[],
|
||||
eosTokenId: number,
|
||||
language = 'english',
|
||||
): [string[], number[][], number[][]] {
|
||||
let words: string[];
|
||||
let wordTokens: number[][];
|
||||
let tokenIndices: number[][];
|
||||
|
||||
if (['chinese', 'japanese', 'thai', 'lao', 'myanmar', 'zh', 'ja', 'th', 'lo', 'my'].includes(language)) {
|
||||
[words, wordTokens, tokenIndices] = splitTokensOnUnicode(tokenizer, tokens);
|
||||
} else {
|
||||
[words, wordTokens, tokenIndices] = splitTokensOnSpaces(tokenizer, tokens, eosTokenId);
|
||||
}
|
||||
|
||||
return mergePunctuations(words, wordTokens, tokenIndices);
|
||||
}
|
||||
|
||||
export function extractTokenStartTimestamps(input: {
|
||||
crossAttentions: Record<string, ort.Tensor>;
|
||||
decoderLayers: number;
|
||||
alignmentHeads: Array<[number, number]>;
|
||||
numFrames: number;
|
||||
numInputIds: number;
|
||||
timePrecision?: number;
|
||||
sequenceLength: number;
|
||||
}): number[] {
|
||||
const {
|
||||
crossAttentions,
|
||||
decoderLayers,
|
||||
alignmentHeads,
|
||||
numFrames,
|
||||
numInputIds,
|
||||
timePrecision = 0.02,
|
||||
sequenceLength,
|
||||
} = input;
|
||||
|
||||
const frameCount = Math.max(1, numFrames);
|
||||
const perLayer: Float32Array[] = [];
|
||||
for (let layer = 0; layer < decoderLayers; layer += 1) {
|
||||
const key = `cross_attentions.${layer}`;
|
||||
const tensor = crossAttentions[key];
|
||||
if (!tensor) continue;
|
||||
perLayer[layer] = tensor.data as Float32Array;
|
||||
}
|
||||
|
||||
const selected: Float32Array[] = [];
|
||||
let seqLen = 0;
|
||||
let attnFrames = 0;
|
||||
for (const [layer, head] of alignmentHeads) {
|
||||
const flat = perLayer[layer];
|
||||
if (!flat) continue;
|
||||
const layerTensor = crossAttentions[`cross_attentions.${layer}`];
|
||||
if (!layerTensor || layerTensor.dims.length < 4) continue;
|
||||
const [, numHeads, currentSeqLen, currentFrames] = layerTensor.dims;
|
||||
if (head >= numHeads) continue;
|
||||
seqLen = currentSeqLen;
|
||||
attnFrames = Math.min(currentFrames, frameCount);
|
||||
const headSlice = new Float32Array(seqLen * attnFrames);
|
||||
for (let s = 0; s < seqLen; s += 1) {
|
||||
for (let f = 0; f < attnFrames; f += 1) {
|
||||
const flatIndex = (((head * currentSeqLen) + s) * currentFrames) + f;
|
||||
headSlice[(s * attnFrames) + f] = flat[flatIndex] ?? 0;
|
||||
}
|
||||
}
|
||||
selected.push(headSlice);
|
||||
}
|
||||
|
||||
if (!selected.length || seqLen === 0 || attnFrames === 0) {
|
||||
return new Array(sequenceLength).fill(0);
|
||||
}
|
||||
|
||||
const normalizedHeads = selected.map((headData) => {
|
||||
const means = new Float32Array(attnFrames);
|
||||
const stds = new Float32Array(attnFrames);
|
||||
|
||||
for (let f = 0; f < attnFrames; f += 1) {
|
||||
let sum = 0;
|
||||
for (let s = 0; s < seqLen; s += 1) sum += headData[(s * attnFrames) + f];
|
||||
const mean = sum / seqLen;
|
||||
means[f] = mean;
|
||||
let varSum = 0;
|
||||
for (let s = 0; s < seqLen; s += 1) {
|
||||
const d = headData[(s * attnFrames) + f] - mean;
|
||||
varSum += d * d;
|
||||
}
|
||||
stds[f] = Math.sqrt(varSum / seqLen) || 1;
|
||||
}
|
||||
|
||||
const out = new Float32Array(headData.length);
|
||||
for (let s = 0; s < seqLen; s += 1) {
|
||||
const row = new Float32Array(attnFrames);
|
||||
for (let f = 0; f < attnFrames; f += 1) {
|
||||
row[f] = (headData[(s * attnFrames) + f] - means[f]) / stds[f];
|
||||
}
|
||||
const filtered = medianFilter(row, 7);
|
||||
out.set(filtered, s * attnFrames);
|
||||
}
|
||||
return out;
|
||||
});
|
||||
|
||||
const croppedRows = Math.max(0, seqLen - numInputIds);
|
||||
if (croppedRows === 0) return new Array(sequenceLength).fill(0);
|
||||
|
||||
const matrix: Float32Array[] = Array.from({ length: croppedRows }, () => new Float32Array(attnFrames));
|
||||
for (const headData of normalizedHeads) {
|
||||
for (let r = 0; r < croppedRows; r += 1) {
|
||||
const srcRow = r + numInputIds;
|
||||
for (let f = 0; f < attnFrames; f += 1) {
|
||||
matrix[r][f] += headData[(srcRow * attnFrames) + f];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const scale = 1 / normalizedHeads.length;
|
||||
for (let r = 0; r < croppedRows; r += 1) {
|
||||
for (let f = 0; f < attnFrames; f += 1) {
|
||||
matrix[r][f] = -matrix[r][f] * scale;
|
||||
}
|
||||
}
|
||||
|
||||
const [textIndices, timeIndices] = dynamicTimeWarping(matrix, croppedRows, attnFrames);
|
||||
const jumps = new Array(textIndices.length).fill(false);
|
||||
for (let i = 0; i < textIndices.length; i += 1) {
|
||||
jumps[i] = i === 0 ? true : textIndices[i] !== textIndices[i - 1];
|
||||
}
|
||||
|
||||
const jumpTimes: number[] = [];
|
||||
for (let i = 0; i < jumps.length; i += 1) {
|
||||
if (jumps[i]) jumpTimes.push(timeIndices[i] * timePrecision);
|
||||
}
|
||||
|
||||
const timestamps = new Array(sequenceLength).fill(0);
|
||||
for (let i = 0; i < numInputIds && i < timestamps.length; i += 1) timestamps[i] = 0;
|
||||
for (let i = 0; i < jumpTimes.length && (numInputIds + i) < timestamps.length; i += 1) {
|
||||
timestamps[numInputIds + i] = jumpTimes[i];
|
||||
}
|
||||
if (timestamps.length > 0 && jumpTimes.length > 0) {
|
||||
timestamps[timestamps.length - 1] = jumpTimes[jumpTimes.length - 1];
|
||||
}
|
||||
return timestamps;
|
||||
}
|
||||
|
||||
export function buildWordsFromTimestampedTokens(input: {
|
||||
tokens: number[];
|
||||
tokenStartTimestamps: number[];
|
||||
tokenizer: Pick<Tokenizer, 'decode'>;
|
||||
eosTokenId: number;
|
||||
promptLength: number;
|
||||
timestampBeginTokenId: number;
|
||||
timePrecision?: number;
|
||||
language?: string;
|
||||
}): WhisperWordTiming[] {
|
||||
const {
|
||||
tokens,
|
||||
tokenStartTimestamps,
|
||||
tokenizer,
|
||||
eosTokenId,
|
||||
promptLength,
|
||||
timestampBeginTokenId,
|
||||
timePrecision = 0.02,
|
||||
language = 'english',
|
||||
} = input;
|
||||
|
||||
const limit = Math.min(tokens.length, tokenStartTimestamps.length);
|
||||
const tokenRanges: TokenTimestamp[] = [];
|
||||
for (let i = 0; i < limit; i += 1) {
|
||||
const start = tokenStartTimestamps[i] ?? 0;
|
||||
const end = i + 1 < limit ? (tokenStartTimestamps[i + 1] ?? (start + timePrecision)) : (start + timePrecision);
|
||||
tokenRanges.push([start, Math.max(start, end)]);
|
||||
}
|
||||
|
||||
const words: WhisperWordTiming[] = [];
|
||||
let segmentStart: number | null = null;
|
||||
let textTokens: number[] = [];
|
||||
let textRanges: TokenTimestamp[] = [];
|
||||
|
||||
const flushSegment = (segmentEnd: number | null) => {
|
||||
if (!textTokens.length) return;
|
||||
const [wordTexts, , tokenIndices] = combineTokensIntoWords(tokenizer, textTokens, eosTokenId, language);
|
||||
for (let i = 0; i < wordTexts.length; i += 1) {
|
||||
const indices = tokenIndices[i];
|
||||
if (!indices.length) continue;
|
||||
const start = textRanges[indices[0]]?.[0] ?? segmentStart ?? 0;
|
||||
const end = textRanges[indices[indices.length - 1]]?.[1] ?? segmentEnd ?? start;
|
||||
const clampedStart = segmentStart == null ? start : Math.max(segmentStart, start);
|
||||
const clampedEndBase = segmentEnd == null ? end : Math.min(segmentEnd, end);
|
||||
const clampedEnd = Math.max(
|
||||
clampedStart + (clampedEndBase <= clampedStart ? timePrecision : 0),
|
||||
clampedEndBase,
|
||||
);
|
||||
words.push({
|
||||
word: wordTexts[i].trim(),
|
||||
startSec: round2(clampedStart),
|
||||
endSec: round2(clampedEnd),
|
||||
});
|
||||
}
|
||||
textTokens = [];
|
||||
textRanges = [];
|
||||
};
|
||||
|
||||
for (let i = promptLength; i < limit; i += 1) {
|
||||
const token = tokens[i];
|
||||
if (token === eosTokenId) break;
|
||||
|
||||
if (token >= timestampBeginTokenId) {
|
||||
const ts = (token - timestampBeginTokenId) * timePrecision;
|
||||
if (segmentStart == null) {
|
||||
segmentStart = ts;
|
||||
} else {
|
||||
flushSegment(ts);
|
||||
segmentStart = ts;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
textTokens.push(token);
|
||||
textRanges.push(tokenRanges[i]);
|
||||
}
|
||||
|
||||
flushSegment(null);
|
||||
return words.filter((w) => w.word.length > 0);
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import { test, expect } from '@playwright/test';
|
||||
import {
|
||||
mapWordsToSentenceOffsets,
|
||||
} from '../../src/lib/server/whisper/alignment-mapping';
|
||||
} from '@openreader/compute-core/whisper';
|
||||
|
||||
test.describe('whisper alignment mapping', () => {
|
||||
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { test, expect } from '@playwright/test';
|
||||
import { readFile } from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { alignAudioWithText } from '../../src/lib/server/whisper/alignment';
|
||||
import { alignAudioWithText } from '@openreader/compute-core/whisper';
|
||||
|
||||
test.describe('whisper alignment smoke', () => {
|
||||
test('runs ONNX alignment end-to-end without decoder reshape errors', async () => {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import path from 'path';
|
|||
import {
|
||||
createSingleflightRunner,
|
||||
ensureWhisperArtifacts,
|
||||
} from '../../src/lib/server/whisper/ensureModel';
|
||||
} from '@openreader/compute-core/whisper';
|
||||
|
||||
function sha256(bytes: Uint8Array): string {
|
||||
return createHash('sha256').update(bytes).digest('hex');
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { test, expect } from '@playwright/test';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '../../src/lib/server/whisper/spectral';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core/whisper';
|
||||
|
||||
function dftPower(samples: Float32Array, k: number): number {
|
||||
const n = samples.length;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
|
|||
import {
|
||||
buildWordsFromTimestampedTokens,
|
||||
extractTokenStartTimestamps,
|
||||
} from '../../src/lib/server/whisper/token-timestamps';
|
||||
} from '@openreader/compute-core/whisper';
|
||||
|
||||
test.describe('whisper token timestamp alignment', () => {
|
||||
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@
|
|||
"@openreader/compute-core/contracts": ["./compute/core/src/contracts.ts"],
|
||||
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"],
|
||||
"@openreader/compute-core/runtime": ["./compute/core/src/runtime/index.ts"],
|
||||
"@openreader/compute-core/whisper": ["./compute/core/src/whisper/index.ts"],
|
||||
"@openreader/compute-core/pdf-layout": ["./compute/core/src/pdf-layout/index.ts"],
|
||||
"@openreader/compute-core/pdf-layout/parse": ["./compute/core/src/pdf-layout/parsePdf.ts"]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue