Marc Mintel
5397309103
Some checks failed
Build & Deploy / 🔍 Prepare (push) Successful in 20s
Build & Deploy / 🧪 QA (push) Failing after 34s
Build & Deploy / 🏗️ Build (push) Has started running
Build & Deploy / 🚀 Deploy (push) Has been cancelled
Build & Deploy / 🧪 Smoke Test (push) Has been cancelled
Build & Deploy / ⚡ Lighthouse (push) Has been cancelled
Build & Deploy / 🔔 Notify (push) Has been cancelled
33 lines
1.6 KiB
Plaintext
33 lines
1.6 KiB
Plaintext
import type { Readable } from 'node:stream';
|
|
import { type AnyWebByteStream } from 'peek-readable';
|
|
import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
|
|
import { BufferTokenizer } from './BufferTokenizer.js';
|
|
import type { ITokenizerOptions } from './types.js';
|
|
export { EndOfStreamError, type AnyWebByteStream } from 'peek-readable';
|
|
export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
|
|
export type { IToken, IGetToken } from '@tokenizer/token';
|
|
export { AbstractTokenizer } from './AbstractTokenizer.js';
|
|
/**
|
|
* Construct ReadStreamTokenizer from given Stream.
|
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
* @param stream - Read from Node.js Stream.Readable
|
|
* @param options - Tokenizer options
|
|
* @returns ReadStreamTokenizer
|
|
*/
|
|
export declare function fromStream(stream: Readable, options?: ITokenizerOptions): ReadStreamTokenizer;
|
|
/**
|
|
* Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
|
|
* Will set fileSize, if provided given Stream has set the .path property/
|
|
* @param webStream - Read from Node.js Stream.Readable (must be a byte stream)
|
|
* @param options - Tokenizer options
|
|
* @returns ReadStreamTokenizer
|
|
*/
|
|
export declare function fromWebStream(webStream: AnyWebByteStream, options?: ITokenizerOptions): ReadStreamTokenizer;
|
|
/**
|
|
* Construct ReadStreamTokenizer from given Buffer.
|
|
* @param uint8Array - Uint8Array to tokenize
|
|
* @param options - Tokenizer options
|
|
* @returns BufferTokenizer
|
|
*/
|
|
export declare function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer;
|