Preserve split UTF-8 request bodies in generated serializers

ChangeLog.md

@@ -10,6 +10,7 @@ General:
 - Fixed `--oauth` startup validation and reporting. Azurite now reports supported OAuth levels consistently across command-line entrypoints and no longer crashes when `--oauth` is specified without a value. (issue #2525)
 - Fixed SharedKey and SharedKeyLite authentication when both `date` and `x-ms-date` headers are present, matching Azure Storage signing behaviour for Blob, Queue, and Table services. (issue #1385)
 - Removed npm and its transitive dependencies from the Linux Docker runtime image while retaining Node.js and existing JavaScript entrypoints, reducing the container attack surface and addressing npm-related vulnerability reports. (issue #2758)
+- Fixed Blob, Queue, and Table request body decoding so multi-byte UTF-8 characters split across incoming chunks are preserved. (issue #2670)
 
 Blob:
 

src/blob/generated/utils/serializer.ts

@@ -182,15 +182,18 @@ export async function deserialize(
 
 async function readRequestIntoText(req: IRequest): Promise<string> {
   return new Promise<string>((resolve, reject) => {
-    const segments: string[] = [];
+    const segments: Buffer[] = [];
     const bodyStream = req.getBodyStream();
     bodyStream.on("data", buffer => {
-      segments.push(buffer);
+      segments.push(
+        typeof buffer === "string"
+          ? Buffer.from(buffer, "utf8")
+          : Buffer.from(buffer)
+      );
     });
     bodyStream.on("error", reject);
     bodyStream.on("end", () => {
-      const joined = segments.join("");
-      resolve(joined);
+      resolve(Buffer.concat(segments).toString("utf8"));
     });
   });
 }

src/queue/generated/utils/serializer.ts

@@ -182,15 +182,18 @@ export async function deserialize(
 
 async function readRequestIntoText(req: IRequest): Promise<string> {
   return new Promise<string>((resolve, reject) => {
-    const segments: string[] = [];
+    const segments: Buffer[] = [];
     const bodyStream = req.getBodyStream();
     bodyStream.on("data", buffer => {
-      segments.push(buffer);
+      segments.push(
+        typeof buffer === "string"
+          ? Buffer.from(buffer, "utf8")
+          : Buffer.from(buffer)
+      );
     });
     bodyStream.on("error", reject);
     bodyStream.on("end", () => {
-      const joined = segments.join("");
-      resolve(joined);
+      resolve(Buffer.concat(segments).toString("utf8"));
     });
   });
 }

src/table/generated/utils/serializer.ts

@@ -182,15 +182,18 @@ export async function deserialize(
 
 async function readRequestIntoText(req: IRequest): Promise<string> {
   return new Promise<string>((resolve, reject) => {
-    const segments: string[] = [];
+    const segments: Buffer[] = [];
     const bodyStream = req.getBodyStream();
     bodyStream.on("data", buffer => {
-      segments.push(buffer);
+      segments.push(
+        typeof buffer === "string"
+          ? Buffer.from(buffer, "utf8")
+          : Buffer.from(buffer)
+      );
     });
     bodyStream.on("error", reject);
     bodyStream.on("end", () => {
-      const joined = segments.join("");
-      resolve(joined);
+      resolve(Buffer.concat(segments).toString("utf8"));
     });
   });
 }

tests/common/generatedSerializer.test.ts

@@ -0,0 +1,144 @@
+import { strict as assert } from "assert";
+import { Readable } from "stream";
+
+import * as msRest from "@azure/ms-rest-js";
+
+import BlobContext from "../../src/blob/generated/Context";
+import BlobIRequest from "../../src/blob/generated/IRequest";
+import BlobILogger from "../../src/blob/generated/utils/ILogger";
+import { deserialize as deserializeBlob } from "../../src/blob/generated/utils/serializer";
+import QueueContext from "../../src/queue/generated/Context";
+import QueueIRequest from "../../src/queue/generated/IRequest";
+import QueueILogger from "../../src/queue/generated/utils/ILogger";
+import { deserialize as deserializeQueue } from "../../src/queue/generated/utils/serializer";
+import TableContext from "../../src/table/generated/Context";
+import TableIRequest from "../../src/table/generated/IRequest";
+import TableILogger from "../../src/table/generated/utils/ILogger";
+import { deserialize as deserializeTable } from "../../src/table/generated/utils/serializer";
+
+const requestBodyMapper: msRest.Mapper = {
+  serializedName: "body",
+  type: {
+    name: "String"
+  }
+};
+
+const spec = {
+  httpMethod: "POST",
+  path: "/",
+  urlParameters: [],
+  requestBody: {
+    parameterPath: "bodyParameter",
+    mapper: requestBodyMapper
+  },
+  responses: {},
+  serializer: new msRest.Serializer()
+} as msRest.OperationSpec;
+
+const logger: BlobILogger & QueueILogger & TableILogger = {
+  error: () => undefined,
+  warn: () => undefined,
+  info: () => undefined,
+  verbose: () => undefined,
+  debug: () => undefined
+};
+
+type DeserializeCase = {
+  name: string;
+  deserialize: (
+    context: any,
+    req: BlobIRequest & QueueIRequest & TableIRequest,
+    operationSpec: msRest.OperationSpec,
+    operationLogger: BlobILogger & QueueILogger & TableILogger
+  ) => Promise<unknown>;
+  createContext: () => BlobContext | QueueContext | TableContext;
+};
+
+describe("Generated serializer readRequestIntoText @loki", () => {
+  const cases: DeserializeCase[] = [
+    {
+      name: "Blob",
+      deserialize: deserializeBlob,
+      createContext: () =>
+        new BlobContext({ generated: { contextID: "blob" } }, "generated")
+    },
+    {
+      name: "Queue",
+      deserialize: deserializeQueue,
+      createContext: () =>
+        new QueueContext({ generated: { contextID: "queue" } }, "generated")
+    },
+    {
+      name: "Table",
+      deserialize: deserializeTable,
+      createContext: () =>
+        new TableContext({ generated: { contextID: "table" } }, "generated")
+    }
+  ];
+
+  for (const testCase of cases) {
+    it(`${testCase.name} preserves multi-byte UTF-8 characters split across chunks`, async () => {
+      const expectedBodyParameter = "before 🌊 after";
+      const request = createRequestWithSplitMultiByteCharacter(
+        JSON.stringify(expectedBodyParameter),
+        "🌊"
+      );
+
+      const parameters = (await testCase.deserialize(
+        testCase.createContext(),
+        request,
+        spec,
+        logger
+      )) as { bodyParameter: string; body: string };
+
+      assert.equal(parameters.bodyParameter, expectedBodyParameter);
+      assert.equal(parameters.body, JSON.stringify(expectedBodyParameter));
+    });
+  }
+});
+
+function createRequestWithSplitMultiByteCharacter(
+  body: string,
+  splitCharacter: string
+): BlobIRequest & QueueIRequest & TableIRequest {
+  const bodyBuffer = Buffer.from(body, "utf8");
+  const splitCharacterBuffer = Buffer.from(splitCharacter, "utf8");
+  assert.ok(
+    splitCharacterBuffer.length > 1,
+    "splitCharacter must be a multi-byte character"
+  );
+  const splitCharacterIndex = bodyBuffer.indexOf(splitCharacterBuffer);
+  assert.notEqual(
+    splitCharacterIndex,
+    -1,
+    "splitCharacter must appear in body"
+  );
+  const splitIndex =
+    splitCharacterIndex + Math.floor(splitCharacterBuffer.length / 2);
+  let storedBody: string | undefined;
+
+  const request = {
+    getMethod: () => "POST",
+    getUrl: () => "/",
+    getEndpoint: () => "/",
+    getPath: () => "/",
+    getBodyStream: () =>
+      Readable.from([
+        bodyBuffer.subarray(0, splitIndex),
+        bodyBuffer.subarray(splitIndex)
+      ]),
+    setBody: (newBody: string | undefined) => {
+      storedBody = newBody;
+      return request;
+    },
+    getBody: () => storedBody,
+    getHeader: (field: string) =>
+      field.toLowerCase() === "content-type" ? "application/json" : undefined,
+    getHeaders: () => ({ "content-type": "application/json" }),
+    getRawHeaders: () => [],
+    getQuery: () => undefined,
+    getProtocol: () => "http"
+  } as BlobIRequest & QueueIRequest & TableIRequest;
+
+  return request;
+}