perf: Parse.Query.include now fetches pointers at same level in parallel (#9861)

2025-11-17 15:42:49 +01:00
parent 52f7c89f3f
commit dafea21eb3
4 changed files with 279 additions and 81 deletions
--- a/.github/workflows/ci-performance.yml
+++ b/.github/workflows/ci-performance.yml
@@ -70,7 +70,7 @@ jobs:
        env:
          NODE_ENV: production
        run: |
-          echo "Running baseline benchmarks with CPU affinity (using PR's benchmark script)..."
+          echo "Running baseline benchmarks..."
          if [ ! -f "benchmark/performance.js" ]; then
            echo "⚠️ Benchmark script not found - this is expected for new features"
            echo "Skipping baseline benchmark"
@@ -135,7 +135,7 @@ jobs:
        env:
          NODE_ENV: production
        run: |
-          echo "Running PR benchmarks with CPU affinity..."
+          echo "Running PR benchmarks..."
          taskset -c 0 npm run benchmark > pr-output.txt 2>&1 || npm run benchmark > pr-output.txt 2>&1 || true
          echo "Benchmark command completed with exit code: $?"
          echo "Output file size: $(wc -c < pr-output.txt) bytes"
--- a/benchmark/performance.js
+++ b/benchmark/performance.js
@@ -200,11 +200,11 @@ async function measureOperation({ name, operation, iterations, skipWarmup = fals
 /**
 * Benchmark: Object Create
 */
-async function benchmarkObjectCreate() {
+async function benchmarkObjectCreate(name) {
  let counter = 0;

  return measureOperation({
-    name: 'Object Create',
+    name,
    iterations: 1_000,
    operation: async () => {
      const TestObject = Parse.Object.extend('BenchmarkTest');
@@ -220,7 +220,7 @@ async function benchmarkObjectCreate() {
 /**
 * Benchmark: Object Read (by ID)
 */
-async function benchmarkObjectRead() {
+async function benchmarkObjectRead(name) {
  // Setup: Create test objects
  const TestObject = Parse.Object.extend('BenchmarkTest');
  const objects = [];
@@ -236,7 +236,7 @@ async function benchmarkObjectRead() {
  let counter = 0;

  return measureOperation({
-    name: 'Object Read',
+    name,
    iterations: 1_000,
    operation: async () => {
      const query = new Parse.Query('BenchmarkTest');
@@ -248,7 +248,7 @@ async function benchmarkObjectRead() {
 /**
 * Benchmark: Object Update
 */
-async function benchmarkObjectUpdate() {
+async function benchmarkObjectUpdate(name) {
  // Setup: Create test objects
  const TestObject = Parse.Object.extend('BenchmarkTest');
  const objects = [];
@@ -265,7 +265,7 @@ async function benchmarkObjectUpdate() {
  let counter = 0;

  return measureOperation({
-    name: 'Object Update',
+    name,
    iterations: 1_000,
    operation: async () => {
      const obj = objects[counter++ % objects.length];
@@ -279,7 +279,7 @@ async function benchmarkObjectUpdate() {
 /**
 * Benchmark: Simple Query
 */
-async function benchmarkSimpleQuery() {
+async function benchmarkSimpleQuery(name) {
  // Setup: Create test data
  const TestObject = Parse.Object.extend('BenchmarkTest');
  const objects = [];
@@ -296,7 +296,7 @@ async function benchmarkSimpleQuery() {
  let counter = 0;

  return measureOperation({
-    name: 'Simple Query',
+    name,
    iterations: 1_000,
    operation: async () => {
      const query = new Parse.Query('BenchmarkTest');
@@ -309,11 +309,11 @@ async function benchmarkSimpleQuery() {
 /**
 * Benchmark: Batch Save (saveAll)
 */
-async function benchmarkBatchSave() {
+async function benchmarkBatchSave(name) {
  const BATCH_SIZE = 10;

  return measureOperation({
-    name: 'Batch Save (10 objects)',
+    name,
    iterations: 1_000,
    operation: async () => {
      const TestObject = Parse.Object.extend('BenchmarkTest');
@@ -334,11 +334,11 @@ async function benchmarkBatchSave() {
 /**
 * Benchmark: User Signup
 */
-async function benchmarkUserSignup() {
+async function benchmarkUserSignup(name) {
  let counter = 0;

  return measureOperation({
-    name: 'User Signup',
+    name,
    iterations: 500,
    operation: async () => {
      counter++;
@@ -354,7 +354,7 @@ async function benchmarkUserSignup() {
 /**
 * Benchmark: User Login
 */
-async function benchmarkUserLogin() {
+async function benchmarkUserLogin(name) {
  // Setup: Create test users
  const users = [];

@@ -371,7 +371,7 @@ async function benchmarkUserLogin() {
  let counter = 0;

  return measureOperation({
-    name: 'User Login',
+    name,
    iterations: 500,
    operation: async () => {
      const userCreds = users[counter++ % users.length];
@@ -382,52 +382,146 @@ async function benchmarkUserLogin() {
 }

 /**
- * Benchmark: Query with Include (Parallel Include Pointers)
+ * Benchmark: Query with Include (Parallel Pointers)
+ * Tests the performance improvement when fetching multiple pointers at the same level.
 */
-async function benchmarkQueryWithInclude() {
-  // Setup: Create nested object hierarchy
+async function benchmarkQueryWithIncludeParallel(name) {
+  const PointerAClass = Parse.Object.extend('PointerA');
+  const PointerBClass = Parse.Object.extend('PointerB');
+  const PointerCClass = Parse.Object.extend('PointerC');
+  const RootClass = Parse.Object.extend('Root');
+
+  // Create pointer objects
+  const pointerAObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new PointerAClass();
+    obj.set('name', `pointerA-${i}`);
+    pointerAObjects.push(obj);
+  }
+  await Parse.Object.saveAll(pointerAObjects);
+
+  const pointerBObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new PointerBClass();
+    obj.set('name', `pointerB-${i}`);
+    pointerBObjects.push(obj);
+  }
+  await Parse.Object.saveAll(pointerBObjects);
+
+  const pointerCObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new PointerCClass();
+    obj.set('name', `pointerC-${i}`);
+    pointerCObjects.push(obj);
+  }
+  await Parse.Object.saveAll(pointerCObjects);
+
+  // Create Root objects with multiple pointers at the same level
+  const rootObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new RootClass();
+    obj.set('name', `root-${i}`);
+    obj.set('pointerA', pointerAObjects[i % pointerAObjects.length]);
+    obj.set('pointerB', pointerBObjects[i % pointerBObjects.length]);
+    obj.set('pointerC', pointerCObjects[i % pointerCObjects.length]);
+    rootObjects.push(obj);
+  }
+  await Parse.Object.saveAll(rootObjects);
+
+  return measureOperation({
+    name,
+    skipWarmup: true,
+    dbLatency: 100,
+    iterations: 100,
+    operation: async () => {
+      const query = new Parse.Query('Root');
+      // Include multiple pointers at the same level - should fetch in parallel
+      query.include(['pointerA', 'pointerB', 'pointerC']);
+      await query.find();
+    },
+  });
+}
+
+/**
+ * Benchmark: Query with Include (Nested Pointers with Parallel Leaf Nodes)
+ * Tests the PR's optimization for parallel fetching at each nested level.
+ * Pattern: p1.p2.p3, p1.p2.p4, p1.p2.p5
+ * After fetching p2, we know the objectIds and can fetch p3, p4, p5 in parallel.
+ */
+async function benchmarkQueryWithIncludeNested(name) {
+  const Level3AClass = Parse.Object.extend('Level3A');
+  const Level3BClass = Parse.Object.extend('Level3B');
+  const Level3CClass = Parse.Object.extend('Level3C');
  const Level2Class = Parse.Object.extend('Level2');
  const Level1Class = Parse.Object.extend('Level1');
  const RootClass = Parse.Object.extend('Root');

+  // Create Level3 objects (leaf nodes)
+  const level3AObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new Level3AClass();
+    obj.set('name', `level3A-${i}`);
+    level3AObjects.push(obj);
+  }
+  await Parse.Object.saveAll(level3AObjects);
+
+  const level3BObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new Level3BClass();
+    obj.set('name', `level3B-${i}`);
+    level3BObjects.push(obj);
+  }
+  await Parse.Object.saveAll(level3BObjects);
+
+  const level3CObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new Level3CClass();
+    obj.set('name', `level3C-${i}`);
+    level3CObjects.push(obj);
+  }
+  await Parse.Object.saveAll(level3CObjects);
+
+  // Create Level2 objects pointing to multiple Level3 objects
+  const level2Objects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new Level2Class();
+    obj.set('name', `level2-${i}`);
+    obj.set('level3A', level3AObjects[i % level3AObjects.length]);
+    obj.set('level3B', level3BObjects[i % level3BObjects.length]);
+    obj.set('level3C', level3CObjects[i % level3CObjects.length]);
+    level2Objects.push(obj);
+  }
+  await Parse.Object.saveAll(level2Objects);
+
+  // Create Level1 objects pointing to Level2
+  const level1Objects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new Level1Class();
+    obj.set('name', `level1-${i}`);
+    obj.set('level2', level2Objects[i % level2Objects.length]);
+    level1Objects.push(obj);
+  }
+  await Parse.Object.saveAll(level1Objects);
+
+  // Create Root objects pointing to Level1
+  const rootObjects = [];
+  for (let i = 0; i < 10; i++) {
+    const obj = new RootClass();
+    obj.set('name', `root-${i}`);
+    obj.set('level1', level1Objects[i % level1Objects.length]);
+    rootObjects.push(obj);
+  }
+  await Parse.Object.saveAll(rootObjects);
+
  return measureOperation({
-    name: 'Query with Include (2 levels)',
+    name,
    skipWarmup: true,
-    dbLatency: 50,
+    dbLatency: 100,
    iterations: 100,
    operation: async () => {
-      // Create 10 Level2 objects
-      const level2Objects = [];
-      for (let i = 0; i < 10; i++) {
-        const obj = new Level2Class();
-        obj.set('name', `level2-${i}`);
-        obj.set('value', i);
-        level2Objects.push(obj);
-      }
-      await Parse.Object.saveAll(level2Objects);
-
-      // Create 10 Level1 objects, each pointing to a Level2 object
-      const level1Objects = [];
-      for (let i = 0; i < 10; i++) {
-        const obj = new Level1Class();
-        obj.set('name', `level1-${i}`);
-        obj.set('level2', level2Objects[i % level2Objects.length]);
-        level1Objects.push(obj);
-      }
-      await Parse.Object.saveAll(level1Objects);
-
-      // Create 10 Root objects, each pointing to a Level1 object
-      const rootObjects = [];
-      for (let i = 0; i < 10; i++) {
-        const obj = new RootClass();
-        obj.set('name', `root-${i}`);
-        obj.set('level1', level1Objects[i % level1Objects.length]);
-        rootObjects.push(obj);
-      }
-      await Parse.Object.saveAll(rootObjects);
-
      const query = new Parse.Query('Root');
-      query.include('level1.level2');
+      // After fetching level1.level2, the PR should fetch level3A, level3B, level3C in parallel
+      query.include(['level1.level2.level3A', 'level1.level2.level3B', 'level1.level2.level3C']);
      await query.find();
    },
  });
@@ -453,14 +547,15 @@ async function runBenchmarks() {

    // Define all benchmarks to run
    const benchmarks = [
-      { name: 'Object Create', fn: benchmarkObjectCreate },
-      { name: 'Object Read', fn: benchmarkObjectRead },
-      { name: 'Object Update', fn: benchmarkObjectUpdate },
-      { name: 'Simple Query', fn: benchmarkSimpleQuery },
-      { name: 'Batch Save', fn: benchmarkBatchSave },
-      { name: 'User Signup', fn: benchmarkUserSignup },
-      { name: 'User Login', fn: benchmarkUserLogin },
-      { name: 'Query with Include', fn: benchmarkQueryWithInclude },
+      { name: 'Object.save (create)', fn: benchmarkObjectCreate },
+      { name: 'Object.save (update)', fn: benchmarkObjectUpdate },
+      { name: 'Object.saveAll (batch save)', fn: benchmarkBatchSave },
+      { name: 'Query.get (by objectId)', fn: benchmarkObjectRead },
+      { name: 'Query.find (simple query)', fn: benchmarkSimpleQuery },
+      { name: 'User.signUp', fn: benchmarkUserSignup },
+      { name: 'User.login', fn: benchmarkUserLogin },
+      { name: 'Query.include (parallel pointers)', fn: benchmarkQueryWithIncludeParallel },
+      { name: 'Query.include (nested pointers)', fn: benchmarkQueryWithIncludeNested },
    ];

    // Run each benchmark with database cleanup
@@ -468,7 +563,7 @@ async function runBenchmarks() {
      logInfo(`\nRunning benchmark '${benchmark.name}'...`);
      resetParseServer();
      await cleanupDatabase();
-      results.push(await benchmark.fn());
+      results.push(await benchmark.fn(benchmark.name));
    }

    // Output results in github-action-benchmark format (stdout)
--- a/spec/RestQuery.spec.js
+++ b/spec/RestQuery.spec.js
@@ -386,6 +386,88 @@ describe('rest query', () => {
        }
      );
  });
+
+  it('battle test parallel include with 100 nested includes', async () => {
+    const RootObject = Parse.Object.extend('RootObject');
+    const Level1Object = Parse.Object.extend('Level1Object');
+    const Level2Object = Parse.Object.extend('Level2Object');
+
+    // Create 100 level2 objects (10 per level1 object)
+    const level2Objects = [];
+    for (let i = 0; i < 100; i++) {
+      const level2 = new Level2Object({
+        index: i,
+        value: `level2_${i}`,
+      });
+      level2Objects.push(level2);
+    }
+    await Parse.Object.saveAll(level2Objects);
+
+    // Create 10 level1 objects, each with 10 pointers to level2 objects
+    const level1Objects = [];
+    for (let i = 0; i < 10; i++) {
+      const level1 = new Level1Object({
+        index: i,
+        value: `level1_${i}`,
+      });
+      // Set 10 pointer fields (level2_0 through level2_9)
+      for (let j = 0; j < 10; j++) {
+        level1.set(`level2_${j}`, level2Objects[i * 10 + j]);
+      }
+      level1Objects.push(level1);
+    }
+    await Parse.Object.saveAll(level1Objects);
+
+    // Create 1 root object with 10 pointers to level1 objects
+    const rootObject = new RootObject({
+      value: 'root',
+    });
+    for (let i = 0; i < 10; i++) {
+      rootObject.set(`level1_${i}`, level1Objects[i]);
+    }
+    await rootObject.save();
+
+    // Build include paths: level1_0 through level1_9, and level1_0.level2_0 through level1_9.level2_9
+    const includePaths = [];
+    for (let i = 0; i < 10; i++) {
+      includePaths.push(`level1_${i}`);
+      for (let j = 0; j < 10; j++) {
+        includePaths.push(`level1_${i}.level2_${j}`);
+      }
+    }
+
+    // Query with all includes
+    const query = new Parse.Query(RootObject);
+    query.equalTo('objectId', rootObject.id);
+    for (const path of includePaths) {
+      query.include(path);
+    }
+    console.time('query.find');
+    const results = await query.find();
+    console.timeEnd('query.find');
+    expect(results.length).toBe(1);
+
+    const result = results[0];
+    expect(result.id).toBe(rootObject.id);
+
+    // Verify all 10 level1 objects are included
+    for (let i = 0; i < 10; i++) {
+      const level1Field = result.get(`level1_${i}`);
+      expect(level1Field).toBeDefined();
+      expect(level1Field instanceof Parse.Object).toBe(true);
+      expect(level1Field.get('index')).toBe(i);
+      expect(level1Field.get('value')).toBe(`level1_${i}`);
+
+      // Verify all 10 level2 objects are included for each level1 object
+      for (let j = 0; j < 10; j++) {
+        const level2Field = level1Field.get(`level2_${j}`);
+        expect(level2Field).toBeDefined();
+        expect(level2Field instanceof Parse.Object).toBe(true);
+        expect(level2Field.get('index')).toBe(i * 10 + j);
+        expect(level2Field.get('value')).toBe(`level2_${i * 10 + j}`);
+      }
+    }
+  });
 });

 describe('RestQuery.each', () => {
--- a/src/RestQuery.js
+++ b/src/RestQuery.js
@@ -856,31 +856,54 @@ _UnsafeRestQuery.prototype.handleExcludeKeys = function () {
 };

 // Augments this.response with data at the paths provided in this.include.
-_UnsafeRestQuery.prototype.handleInclude = function () {
+_UnsafeRestQuery.prototype.handleInclude = async function () {
  if (this.include.length == 0) {
    return;
  }

-  var pathResponse = includePath(
-    this.config,
-    this.auth,
-    this.response,
-    this.include[0],
-    this.context,
-    this.restOptions
-  );
-  if (pathResponse.then) {
-    return pathResponse.then(newResponse => {
-      this.response = newResponse;
-      this.include = this.include.slice(1);
-      return this.handleInclude();
+  const indexedResults = this.response.results.reduce((indexed, result, i) => {
+    indexed[result.objectId] = i;
+    return indexed;
+  }, {});
+
+  // Build the execution tree
+  const executionTree = {}
+  this.include.forEach(path => {
+    let current = executionTree;
+    path.forEach((node) => {
+      if (!current[node]) {
+        current[node] = {
+          path,
+          children: {}
+        };
+      }
+      current = current[node].children
    });
-  } else if (this.include.length > 0) {
-    this.include = this.include.slice(1);
-    return this.handleInclude();
+  });
+
+  const recursiveExecutionTree = async (treeNode) => {
+    const { path, children } = treeNode;
+    const pathResponse = includePath(
+      this.config,
+      this.auth,
+      this.response,
+      path,
+      this.context,
+      this.restOptions,
+      this,
+    );
+    if (pathResponse.then) {
+      const newResponse = await pathResponse
+      newResponse.results.forEach(newObject => {
+        // We hydrate the root of each result with sub results
+        this.response.results[indexedResults[newObject.objectId]][path[0]] = newObject[path[0]];
+      })
+    }
+    return Promise.all(Object.values(children).map(recursiveExecutionTree));
  }

-  return pathResponse;
+  await Promise.all(Object.values(executionTree).map(recursiveExecutionTree));
+  this.include = []
 };

 //Returns a promise of a processed set of results
@@ -1018,7 +1041,6 @@ function includePath(config, auth, response, path, context, restOptions = {}) {
  } else if (restOptions.readPreference) {
    includeRestOptions.readPreference = restOptions.readPreference;
  }
-
  const queryPromises = Object.keys(pointersHash).map(async className => {
    const objectIds = Array.from(pointersHash[className]);
    let where;
@@ -1057,7 +1079,6 @@ function includePath(config, auth, response, path, context, restOptions = {}) {
      }
      return replace;
    }, {});
-
    var resp = {
      results: replacePointers(response.results, path, replace),
    };