Mirror: The highly customizable and versatile GraphQL client with which you add on features like normalized caching as you grow.

fix(core): Fix multibyte character decoding by using `stream` option (#3767)

Co-authored-by: Phil Pluckthun <phil@kitten.sh>

authored by i110 Phil Pluckthun and committed by GitHub 53b822bc 9bb7ef8f

Changed files
+28 -18
.changeset
packages
core
src
internal
+5
.changeset/nervous-flies-confess.md
··· 1 + --- 2 + '@urql/core': patch 3 + --- 4 + 5 + Fix `fetchSource` not text-decoding response chunks as streams, which could cause UTF-8 decoding to break.
+23 -18
packages/core/src/internal/fetchSource.ts
··· 26 26 * The implementation in this file needs to make certain accommodations for: 27 27 * - The Web Fetch API 28 28 * - Non-browser or polyfill Fetch APIs 29 - * - Node.js-like Fetch implementations (see `toString` below) 29 + * - Node.js-like Fetch implementations 30 30 * 31 31 * GraphQL over SSE has a reference implementation, which supports non-HTTP/2 32 32 * modes and is a faithful implementation of the spec. ··· 47 47 import type { Operation, OperationResult, ExecutionResult } from '../types'; 48 48 import { makeResult, makeErrorResult, mergeResultPatch } from '../utils'; 49 49 50 - const decoder = typeof TextDecoder !== 'undefined' ? new TextDecoder() : null; 51 50 const boundaryHeaderRe = /boundary="?([^=";]+)"?/i; 52 51 const eventStreamRe = /data: ?([^\n]+)/; 53 52 54 53 type ChunkData = Buffer | Uint8Array; 55 54 56 - // NOTE: We're avoiding referencing the `Buffer` global here to prevent 57 - // auto-polyfilling in Webpack 58 - const toString = (input: Buffer | ArrayBuffer): string => 59 - input.constructor.name === 'Buffer' 60 - ? (input as Buffer).toString() 61 - : decoder!.decode(input as ArrayBuffer); 62 - 63 - async function* streamBody(response: Response): AsyncIterableIterator<string> { 55 + async function* streamBody( 56 + response: Response 57 + ): AsyncIterableIterator<ChunkData> { 64 58 if (response.body![Symbol.asyncIterator]) { 65 - for await (const chunk of response.body! as any) 66 - yield toString(chunk as ChunkData); 59 + for await (const chunk of response.body! as any) yield chunk as ChunkData; 67 60 } else { 68 61 const reader = response.body!.getReader(); 69 62 let result: ReadableStreamReadResult<ChunkData>; 70 63 try { 71 - while (!(result = await reader.read()).done) yield toString(result.value); 64 + while (!(result = await reader.read()).done) yield result.value; 72 65 } finally { 73 66 reader.cancel(); 74 67 } 75 68 } 76 69 } 77 70 78 - async function* split( 79 - chunks: AsyncIterableIterator<string>, 71 + async function* streamToBoundedChunks( 72 + chunks: AsyncIterableIterator<ChunkData>, 80 73 boundary: string 81 74 ): AsyncIterableIterator<string> { 75 + const decoder = typeof TextDecoder !== 'undefined' ? new TextDecoder() : null; 82 76 let buffer = ''; 83 77 let boundaryIndex: number; 84 78 for await (const chunk of chunks) { 85 - buffer += chunk; 79 + // NOTE: We're avoiding referencing the `Buffer` global here to prevent 80 + // auto-polyfilling in Webpack 81 + buffer += 82 + chunk.constructor.name === 'Buffer' 83 + ? (chunk as Buffer).toString() 84 + : decoder!.decode(chunk as ArrayBuffer, { stream: true }); 86 85 while ((boundaryIndex = buffer.indexOf(boundary)) > -1) { 87 86 yield buffer.slice(0, boundaryIndex); 88 87 buffer = buffer.slice(boundaryIndex + boundary.length); ··· 100 99 response: Response 101 100 ): AsyncIterableIterator<ExecutionResult> { 102 101 let payload: any; 103 - for await (const chunk of split(streamBody(response), '\n\n')) { 102 + for await (const chunk of streamToBoundedChunks( 103 + streamBody(response), 104 + '\n\n' 105 + )) { 104 106 const match = chunk.match(eventStreamRe); 105 107 if (match) { 106 108 const chunk = match[1]; ··· 125 127 const boundary = '--' + (boundaryHeader ? boundaryHeader[1] : '-'); 126 128 let isPreamble = true; 127 129 let payload: any; 128 - for await (let chunk of split(streamBody(response), '\r\n' + boundary)) { 130 + for await (let chunk of streamToBoundedChunks( 131 + streamBody(response), 132 + '\r\n' + boundary 133 + )) { 129 134 if (isPreamble) { 130 135 isPreamble = false; 131 136 const preambleIndex = chunk.indexOf(boundary);