]>
Commit | Line | Data |
---|---|---|
1d09f67e TL |
1 | // Licensed to the Apache Software Foundation (ASF) under one |
2 | // or more contributor license agreements. See the NOTICE file | |
3 | // distributed with this work for additional information | |
4 | // regarding copyright ownership. The ASF licenses this file | |
5 | // to you under the Apache License, Version 2.0 (the | |
6 | // "License"); you may not use this file except in compliance | |
7 | // with the License. You may obtain a copy of the License at | |
8 | // | |
9 | // http://www.apache.org/licenses/LICENSE-2.0 | |
10 | // | |
11 | // Unless required by applicable law or agreed to in writing, | |
12 | // software distributed under the License is distributed on an | |
13 | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
14 | // KIND, either express or implied. See the License for the | |
15 | // specific language governing permissions and limitations | |
16 | // under the License. | |
17 | ||
18 | import { flatbuffers } from 'flatbuffers'; | |
19 | import { encodeUtf8 } from '../util/utf8'; | |
20 | import ByteBuffer = flatbuffers.ByteBuffer; | |
21 | import { TypedArray, TypedArrayConstructor } from '../interfaces'; | |
22 | import { BigIntArray, BigIntArrayConstructor } from '../interfaces'; | |
23 | import { isPromise, isIterable, isAsyncIterable, isIteratorResult, BigInt64Array, BigUint64Array } from './compat'; | |
24 | ||
25 | /** @ignore */ | |
26 | const SharedArrayBuf = (typeof SharedArrayBuffer !== 'undefined' ? SharedArrayBuffer : ArrayBuffer); | |
27 | ||
28 | /** @ignore */ | |
29 | function collapseContiguousByteRanges(chunks: Uint8Array[]) { | |
30 | const result = chunks[0] ? [chunks[0]] : []; | |
31 | let xOffset: number, yOffset: number, xLen: number, yLen: number; | |
32 | for (let x, y, i = 0, j = 0, n = chunks.length; ++i < n;) { | |
33 | x = result[j]; | |
34 | y = chunks[i]; | |
35 | // continue if x and y don't share the same underlying ArrayBuffer, or if x isn't before y | |
36 | if (!x || !y || x.buffer !== y.buffer || y.byteOffset < x.byteOffset) { | |
37 | y && (result[++j] = y); | |
38 | continue; | |
39 | } | |
40 | ({ byteOffset: xOffset, byteLength: xLen } = x); | |
41 | ({ byteOffset: yOffset, byteLength: yLen } = y); | |
42 | // continue if the byte ranges of x and y aren't contiguous | |
43 | if ((xOffset + xLen) < yOffset || (yOffset + yLen) < xOffset) { | |
44 | y && (result[++j] = y); | |
45 | continue; | |
46 | } | |
47 | result[j] = new Uint8Array(x.buffer, xOffset, yOffset - xOffset + yLen); | |
48 | } | |
49 | return result; | |
50 | } | |
51 | ||
52 | /** @ignore */ | |
53 | export function memcpy<TTarget extends ArrayBufferView, TSource extends ArrayBufferView>(target: TTarget, source: TSource, targetByteOffset = 0, sourceByteLength = source.byteLength) { | |
54 | const targetByteLength = target.byteLength; | |
55 | const dst = new Uint8Array(target.buffer, target.byteOffset, targetByteLength); | |
56 | const src = new Uint8Array(source.buffer, source.byteOffset, Math.min(sourceByteLength, targetByteLength)); | |
57 | dst.set(src, targetByteOffset); | |
58 | return target; | |
59 | } | |
60 | ||
61 | /** @ignore */ | |
62 | export function joinUint8Arrays(chunks: Uint8Array[], size?: number | null): [Uint8Array, Uint8Array[], number] { | |
63 | // collapse chunks that share the same underlying ArrayBuffer and whose byte ranges overlap, | |
64 | // to avoid unnecessarily copying the bytes to do this buffer join. This is a common case during | |
65 | // streaming, where we may be reading partial byte ranges out of the same underlying ArrayBuffer | |
66 | const result = collapseContiguousByteRanges(chunks); | |
67 | const byteLength = result.reduce((x, b) => x + b.byteLength, 0); | |
68 | let source: Uint8Array, sliced: Uint8Array, buffer: Uint8Array | void; | |
69 | let offset = 0, index = -1; | |
70 | const length = Math.min(size || Infinity, byteLength); | |
71 | for (let n = result.length; ++index < n;) { | |
72 | source = result[index]; | |
73 | sliced = source.subarray(0, Math.min(source.length, length - offset)); | |
74 | if (length <= (offset + sliced.length)) { | |
75 | if (sliced.length < source.length) { | |
76 | result[index] = source.subarray(sliced.length); | |
77 | } else if (sliced.length === source.length) { index++; } | |
78 | buffer ? memcpy(buffer, sliced, offset) : (buffer = sliced); | |
79 | break; | |
80 | } | |
81 | memcpy(buffer || (buffer = new Uint8Array(length)), sliced, offset); | |
82 | offset += sliced.length; | |
83 | } | |
84 | return [buffer || new Uint8Array(0), result.slice(index), byteLength - (buffer ? buffer.byteLength : 0)]; | |
85 | } | |
86 | ||
87 | /** @ignore */ | |
88 | export type ArrayBufferViewInput = ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined | | |
89 | IteratorResult<ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined> | | |
90 | ReadableStreamReadResult<ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined> ; | |
91 | ||
92 | /** @ignore */ | |
93 | export function toArrayBufferView<T extends TypedArray>(ArrayBufferViewCtor: TypedArrayConstructor<T>, input: ArrayBufferViewInput): T; | |
94 | export function toArrayBufferView<T extends BigIntArray>(ArrayBufferViewCtor: BigIntArrayConstructor<T>, input: ArrayBufferViewInput): T; | |
95 | export function toArrayBufferView(ArrayBufferViewCtor: any, input: ArrayBufferViewInput) { | |
96 | ||
97 | let value: any = isIteratorResult(input) ? input.value : input; | |
98 | ||
99 | if (value instanceof ArrayBufferViewCtor) { | |
100 | if (ArrayBufferViewCtor === Uint8Array) { | |
101 | // Node's `Buffer` class passes the `instanceof Uint8Array` check, but we need | |
102 | // a real Uint8Array, since Buffer#slice isn't the same as Uint8Array#slice :/ | |
103 | return new ArrayBufferViewCtor(value.buffer, value.byteOffset, value.byteLength); | |
104 | } | |
105 | return value; | |
106 | } | |
107 | if (!value) { return new ArrayBufferViewCtor(0); } | |
108 | if (typeof value === 'string') { value = encodeUtf8(value); } | |
109 | if (value instanceof ArrayBuffer) { return new ArrayBufferViewCtor(value); } | |
110 | if (value instanceof SharedArrayBuf) { return new ArrayBufferViewCtor(value); } | |
111 | if (value instanceof ByteBuffer) { return toArrayBufferView(ArrayBufferViewCtor, value.bytes()); } | |
112 | return !ArrayBuffer.isView(value) ? ArrayBufferViewCtor.from(value) : value.byteLength <= 0 ? new ArrayBufferViewCtor(0) | |
113 | : new ArrayBufferViewCtor(value.buffer, value.byteOffset, value.byteLength / ArrayBufferViewCtor.BYTES_PER_ELEMENT); | |
114 | } | |
115 | ||
116 | /** @ignore */ export const toInt8Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int8Array, input); | |
117 | /** @ignore */ export const toInt16Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int16Array, input); | |
118 | /** @ignore */ export const toInt32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int32Array, input); | |
119 | /** @ignore */ export const toBigInt64Array = (input: ArrayBufferViewInput) => toArrayBufferView(BigInt64Array, input); | |
120 | /** @ignore */ export const toUint8Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint8Array, input); | |
121 | /** @ignore */ export const toUint16Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint16Array, input); | |
122 | /** @ignore */ export const toUint32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint32Array, input); | |
123 | /** @ignore */ export const toBigUint64Array = (input: ArrayBufferViewInput) => toArrayBufferView(BigUint64Array, input); | |
124 | /** @ignore */ export const toFloat32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Float32Array, input); | |
125 | /** @ignore */ export const toFloat64Array = (input: ArrayBufferViewInput) => toArrayBufferView(Float64Array, input); | |
126 | /** @ignore */ export const toUint8ClampedArray = (input: ArrayBufferViewInput) => toArrayBufferView(Uint8ClampedArray, input); | |
127 | ||
128 | /** @ignore */ | |
129 | type ArrayBufferViewIteratorInput = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput; | |
130 | ||
131 | /** @ignore */ | |
132 | const pump = <T extends Iterator<any> | AsyncIterator<any>>(iterator: T) => { iterator.next(); return iterator; }; | |
133 | ||
134 | /** @ignore */ | |
135 | export function* toArrayBufferViewIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewIteratorInput) { | |
136 | ||
137 | const wrap = function*<T>(x: T) { yield x; }; | |
138 | const buffers: Iterable<ArrayBufferViewInput> = | |
139 | (typeof source === 'string') ? wrap(source) | |
140 | : (ArrayBuffer.isView(source)) ? wrap(source) | |
141 | : (source instanceof ArrayBuffer) ? wrap(source) | |
142 | : (source instanceof SharedArrayBuf) ? wrap(source) | |
143 | : !isIterable<ArrayBufferViewInput>(source) ? wrap(source) : source; | |
144 | ||
145 | yield* pump((function* (it: Iterator<ArrayBufferViewInput, any, number | undefined>): Generator<T, void, number | undefined> { | |
146 | let r: IteratorResult<any> = <any> null; | |
147 | do { | |
148 | r = it.next(yield toArrayBufferView(ArrayCtor, r)); | |
149 | } while (!r.done); | |
150 | })(buffers[Symbol.iterator]())); | |
151 | return new ArrayCtor(); | |
152 | } | |
153 | ||
154 | /** @ignore */ export const toInt8ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int8Array, input); | |
155 | /** @ignore */ export const toInt16ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int16Array, input); | |
156 | /** @ignore */ export const toInt32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int32Array, input); | |
157 | /** @ignore */ export const toUint8ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint8Array, input); | |
158 | /** @ignore */ export const toUint16ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint16Array, input); | |
159 | /** @ignore */ export const toUint32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint32Array, input); | |
160 | /** @ignore */ export const toFloat32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Float32Array, input); | |
161 | /** @ignore */ export const toFloat64ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Float64Array, input); | |
162 | /** @ignore */ export const toUint8ClampedArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint8ClampedArray, input); | |
163 | ||
164 | /** @ignore */ | |
165 | type ArrayBufferViewAsyncIteratorInput = AsyncIterable<ArrayBufferViewInput> | Iterable<ArrayBufferViewInput> | PromiseLike<ArrayBufferViewInput> | ArrayBufferViewInput; | |
166 | ||
167 | /** @ignore */ | |
168 | export async function* toArrayBufferViewAsyncIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewAsyncIteratorInput): AsyncGenerator<T, T, number | undefined> { | |
169 | ||
170 | // if a Promise, unwrap the Promise and iterate the resolved value | |
171 | if (isPromise<ArrayBufferViewInput>(source)) { | |
172 | return yield* toArrayBufferViewAsyncIterator(ArrayCtor, await source); | |
173 | } | |
174 | ||
175 | const wrap = async function*<T>(x: T) { yield await x; }; | |
176 | const emit = async function* <T extends Iterable<any>>(source: T) { | |
177 | yield* pump((function*(it: Iterator<any>) { | |
178 | let r: IteratorResult<any> = <any> null; | |
179 | do { | |
180 | r = it.next(yield r?.value); | |
181 | } while (!r.done); | |
182 | })(source[Symbol.iterator]())); | |
183 | }; | |
184 | ||
185 | const buffers: AsyncIterable<ArrayBufferViewInput> = | |
186 | (typeof source === 'string') ? wrap(source) // if string, wrap in an AsyncIterableIterator | |
187 | : (ArrayBuffer.isView(source)) ? wrap(source) // if TypedArray, wrap in an AsyncIterableIterator | |
188 | : (source instanceof ArrayBuffer) ? wrap(source) // if ArrayBuffer, wrap in an AsyncIterableIterator | |
189 | : (source instanceof SharedArrayBuf) ? wrap(source) // if SharedArrayBuffer, wrap in an AsyncIterableIterator | |
190 | : isIterable<ArrayBufferViewInput>(source) ? emit(source) // If Iterable, wrap in an AsyncIterableIterator and compose the `next` values | |
191 | : !isAsyncIterable<ArrayBufferViewInput>(source) ? wrap(source) // If not an AsyncIterable, treat as a sentinel and wrap in an AsyncIterableIterator | |
192 | : source; // otherwise if AsyncIterable, use it | |
193 | ||
194 | yield* pump((async function* (it: AsyncIterator<ArrayBufferViewInput, any, number | undefined>): AsyncGenerator<T, void, number | undefined> { | |
195 | let r: IteratorResult<any> = <any> null; | |
196 | do { | |
197 | r = await it.next(yield toArrayBufferView(ArrayCtor, r)); | |
198 | } while (!r.done); | |
199 | })(buffers[Symbol.asyncIterator]())); | |
200 | return new ArrayCtor(); | |
201 | } | |
202 | ||
203 | /** @ignore */ export const toInt8ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int8Array, input); | |
204 | /** @ignore */ export const toInt16ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int16Array, input); | |
205 | /** @ignore */ export const toInt32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int32Array, input); | |
206 | /** @ignore */ export const toUint8ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint8Array, input); | |
207 | /** @ignore */ export const toUint16ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint16Array, input); | |
208 | /** @ignore */ export const toUint32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint32Array, input); | |
209 | /** @ignore */ export const toFloat32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Float32Array, input); | |
210 | /** @ignore */ export const toFloat64ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Float64Array, input); | |
211 | /** @ignore */ export const toUint8ClampedArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint8ClampedArray, input); | |
212 | ||
213 | /** @ignore */ | |
214 | export function rebaseValueOffsets(offset: number, length: number, valueOffsets: Int32Array) { | |
215 | // If we have a non-zero offset, create a new offsets array with the values | |
216 | // shifted by the start offset, such that the new start offset is 0 | |
217 | if (offset !== 0) { | |
218 | valueOffsets = valueOffsets.slice(0, length + 1); | |
219 | for (let i = -1; ++i <= length;) { | |
220 | valueOffsets[i] += offset; | |
221 | } | |
222 | } | |
223 | return valueOffsets; | |
224 | } | |
225 | ||
226 | /** @ignore */ | |
227 | export function compareArrayLike<T extends ArrayLike<any>>(a: T, b: T) { | |
228 | let i = 0; | |
229 | const n = a.length; | |
230 | if (n !== b.length) { return false; } | |
231 | if (n > 0) { | |
232 | do { if (a[i] !== b[i]) { return false; } } while (++i < n); | |
233 | } | |
234 | return true; | |
235 | } |