New query condition support to match all strings that starts with some other given strings (#3864)

* feat: Convert $regex value to RegExp object

* feat: Add lib folder

* Revert "feat: Add lib folder"

This reverts commit c9dfbcbf699ff220baeb2df5586a944d19808e5e.

* feat: Add $regex test in $all array

* test: Test regex with $all only in MongoDB

* Revert "test: Test regex with $all only in MongoDB"

This reverts commit d7194c7869dee98d924fbc2502593a198385dba1.

* feat: Add tests for containsAllStartingWith

* feat: Add postgres support

Thanks to @dplewis

* feat: Check that all values in $all must be regex or none

* test: Check that $all vaules must be regex or none

* feat: Update tests to use only REST API

* refactor: Move $all regex check to adapter

* feat: Check for valid $all values in progres

* refactor: Update function name

* fix:  Postgres $all values regex checking

* fix: Check starts with as string

* fix: Define contains all regex sql function

* fix: Wrong value check

* fix: Check valid data

* fix: Check regex when there is only one value

* fix: Constains all starting with string returns empty with bad params

* fix: Pass correct regex value

* feat: Add missing tests

* feat: Add missing tests

* feat: Add more tests

* fix: Unify MongoDB and PostgreSQL functionality

* fix: Lint checks

* fix: Test broken

$regex in $all list must be { $regex: "string" }

* test for empty $all

spec/MongoTransform.spec.js

@@ -353,6 +353,41 @@ describe('parseObjectToMongoObjectForCreate', () => {
     expect(output.ts.iso).toEqual('2017-01-18T00:00:00.000Z');
     done();
   });
+
+  it('$regex in $all list', (done) => {
+    const input = {
+      arrayField: {'$all': [{$regex: '^\\Qone\\E'}, {$regex: '^\\Qtwo\\E'}, {$regex: '^\\Qthree\\E'}]},
+    };
+    const outputValue = {
+      arrayField: {'$all': [/^\Qone\E/, /^\Qtwo\E/, /^\Qthree\E/]},
+    };
+
+    const output = transform.transformWhere(null, input);
+    jequal(outputValue.arrayField, output.arrayField);
+    done();
+  });
+
+  it('$regex in $all list must be { $regex: "string" }', (done) => {
+    const input = {
+      arrayField: {'$all': [{$regex: 1}]},
+    };
+
+    expect(() => {
+      transform.transformWhere(null, input)
+    }).toThrow();
+    done();
+  });
+
+  it('all values in $all must be $regex (start with string) or non $regex (start with string)', (done) => {
+    const input = {
+      arrayField: {'$all': [{$regex: '^\\Qone\\E'}, {$unknown: '^\\Qtwo\\E'}]},
+    };
+
+    expect(() => {
+      transform.transformWhere(null, input)
+    }).toThrow();
+    done();
+  });
 });
 
 describe('transformUpdate', () => {

spec/ParseQuery.spec.js

@@ -509,6 +509,251 @@ describe('Parse.Query testing', () => {
     });
   });
 
+  it('containsAllStartingWith should match all strings that starts with string', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+    const object2 = new Parse.Object('Object');
+    object2.set('strings', ['the', 'brown', 'fox', 'jumps']);
+    const object3 = new Parse.Object('Object');
+    object3.set('strings', ['over', 'the', 'lazy', 'dog']);
+
+    const objectList = [object, object2, object3];
+
+    Parse.Object.saveAll(objectList).then((results) => {
+      equal(objectList.length, results.length);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: [
+                {$regex: '\^\\Qthe\\E'},
+                {$regex: '\^\\Qfox\\E'},
+                {$regex: '\^\\Qlazy\\E'}
+              ]
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      })
+        .then(function (results) {
+          equal(results.results.length, 1);
+          arrayContains(results.results, object);
+
+          return require('request-promise').get({
+            url: Parse.serverURL + "/classes/Object",
+            json: {
+              where: {
+                strings: {
+                  $all: [
+                    {$regex: '\^\\Qthe\\E'},
+                    {$regex: '\^\\Qlazy\\E'}
+                  ]
+                }
+              }
+            },
+            headers: {
+              'X-Parse-Application-Id': Parse.applicationId,
+              'X-Parse-Javascript-Key': Parse.javaScriptKey
+            }
+          });
+        })
+        .then(function (results) {
+          equal(results.results.length, 2);
+          arrayContains(results.results, object);
+          arrayContains(results.results, object3);
+
+          return require('request-promise').get({
+            url: Parse.serverURL + "/classes/Object",
+            json: {
+              where: {
+                strings: {
+                  $all: [
+                    {$regex: '\^\\Qhe\\E'},
+                    {$regex: '\^\\Qlazy\\E'}
+                  ]
+                }
+              }
+            },
+            headers: {
+              'X-Parse-Application-Id': Parse.applicationId,
+              'X-Parse-Javascript-Key': Parse.javaScriptKey
+            }
+          });
+        })
+        .then(function (results) {
+          equal(results.results.length, 0);
+
+          done();
+        });
+    });
+  });
+
+  it('containsAllStartingWith values must be all of type starting with regex', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+
+    object.save().then(() => {
+      equal(object.isNew(), false);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: [
+                {$regex: '\^\\Qthe\\E'},
+                {$regex: '\^\\Qlazy\\E'},
+                {$regex: '\^\\Qfox\\E'},
+                {$unknown: /unknown/}
+              ]
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      });
+    })
+      .then(function () {
+      }, function () {
+        done();
+      });
+  });
+
+  it('containsAllStartingWith empty array values should return empty results', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+
+    object.save().then(() => {
+      equal(object.isNew(), false);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: []
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      });
+    })
+      .then(function (results) {
+        equal(results.results.length, 0);
+        done();
+      }, function () {
+      });
+  });
+
+  it('containsAllStartingWith single empty value returns empty results', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+
+    object.save().then(() => {
+      equal(object.isNew(), false);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: [ {} ]
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      });
+    })
+      .then(function (results) {
+        equal(results.results.length, 0);
+        done();
+      }, function () {
+      });
+  });
+
+  it('containsAllStartingWith single regex value should return corresponding matching results', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+    const object2 = new Parse.Object('Object');
+    object2.set('strings', ['the', 'brown', 'fox', 'jumps']);
+    const object3 = new Parse.Object('Object');
+    object3.set('strings', ['over', 'the', 'lazy', 'dog']);
+
+    const objectList = [object, object2, object3];
+
+    Parse.Object.saveAll(objectList).then((results) => {
+      equal(objectList.length, results.length);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: [ {$regex: '\^\\Qlazy\\E'} ]
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      });
+    })
+      .then(function (results) {
+        equal(results.results.length, 2);
+        done();
+      }, function () {
+      });
+  });
+
+  it('containsAllStartingWith single invalid regex returns empty results', (done) => {
+
+    const object = new Parse.Object('Object');
+    object.set('strings', ['the', 'brown', 'lazy', 'fox', 'jumps']);
+
+    object.save().then(() => {
+      equal(object.isNew(), false);
+
+      return require('request-promise').get({
+        url: Parse.serverURL + "/classes/Object",
+        json: {
+          where: {
+            strings: {
+              $all: [ {$unknown: '\^\\Qlazy\\E'} ]
+            }
+          }
+        },
+        headers: {
+          'X-Parse-Application-Id': Parse.applicationId,
+          'X-Parse-Javascript-Key': Parse.javaScriptKey
+        }
+      });
+    })
+      .then(function (results) {
+        equal(results.results.length, 0);
+        done();
+      }, function () {
+      });
+  });
+
   const BoxedNumber = Parse.Object.extend({
     className: "BoxedNumber"
   });

src/Adapters/Storage/Mongo/MongoTransform.js

@@ -123,6 +123,44 @@ const transformKeyValueForUpdate = (className, restKey, restValue, parseFormatSc
   return {key, value};
 }
 
+const isRegex = value => {
+  return value && (value instanceof RegExp)
+}
+
+const isStartsWithRegex = value => {
+  if (!isRegex(value)) {
+    return false;
+  }
+
+  const matches = value.toString().match(/\/\^\\Q.*\\E\//);
+  return !!matches;
+}
+
+const isAllValuesRegexOrNone = values => {
+  if (!values || !Array.isArray(values) || values.length === 0) {
+    return true;
+  }
+
+  const firstValuesIsRegex = isStartsWithRegex(values[0]);
+  if (values.length === 1) {
+    return firstValuesIsRegex;
+  }
+
+  for (let i = 1, length = values.length; i < length; ++i) {
+    if (firstValuesIsRegex !== isStartsWithRegex(values[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+const isAnyValueRegex = values => {
+  return values.some(function (value) {
+    return isRegex(value);
+  });
+}
+
 const transformInteriorValue = restValue => {
   if (restValue !== null && typeof restValue === 'object' && Object.keys(restValue).some(key => key.includes('$') || key.includes('.'))) {
     throw new Parse.Error(Parse.Error.INVALID_NESTED_KEY, "Nested keys should not contain the '$' or '.' characters");
@@ -469,6 +507,8 @@ const transformInteriorAtom = (atom) => {
     return DateCoder.JSONToDatabase(atom);
   } else if (BytesCoder.isValidJSON(atom)) {
     return BytesCoder.JSONToDatabase(atom);
+  } else if (typeof atom === 'object' && atom && atom.$regex !== undefined) {
+    return new RegExp(atom.$regex);
   } else {
     return atom;
   }
@@ -740,6 +780,13 @@ function transformConstraint(constraint, field) {
           'bad ' + key + ' value');
       }
       answer[key] = arr.map(transformInteriorAtom);
+
+      const values = answer[key];
+      if (isAnyValueRegex(values) && !isAllValuesRegexOrNone(values)) {
+        throw new Parse.Error(Parse.Error.INVALID_JSON, 'All $all values must be of regex type or none: '
+          + values);
+      }
+
       break;
     }
     case '$regex':

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -418,7 +418,20 @@ const buildWhereClause = ({ schema, query, index }): WhereClause => {
     }
 
     if (Array.isArray(fieldValue.$all) && isArrayField) {
-      patterns.push(`array_contains_all($${index}:name, $${index + 1}::jsonb)`);
+      if (isAnyValueRegexStartsWith(fieldValue.$all)) {
+        if (!isAllValuesRegexOrNone(fieldValue.$all)) {
+          throw new Parse.Error(Parse.Error.INVALID_JSON, 'All $all values must be of regex type or none: '
+            + fieldValue.$all);
+        }
+
+        for (let i = 0; i < fieldValue.$all.length; i += 1) {
+          const value = processRegexPattern(fieldValue.$all[i].$regex);
+          fieldValue.$all[i] = value.substring(1) + '%';
+        }
+        patterns.push(`array_contains_all_regex($${index}:name, $${index + 1}::jsonb)`);
+      } else {
+        patterns.push(`array_contains_all($${index}:name, $${index + 1}::jsonb)`);
+      }
       values.push(fieldName, JSON.stringify(fieldValue.$all));
       index += 2;
     }
@@ -1758,6 +1771,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
             t.none(sql.array.addUnique),
             t.none(sql.array.remove),
             t.none(sql.array.containsAll),
+            t.none(sql.array.containsAllRegex),
             t.none(sql.array.contains)
           ]);
         });
@@ -1862,6 +1876,40 @@ function processRegexPattern(s) {
   return literalizeRegexPart(s);
 }
 
+function isStartsWithRegex(value) {
+  if (!value || typeof value !== 'string' || !value.startsWith('^')) {
+    return false;
+  }
+
+  const matches = value.match(/\^\\Q.*\\E/);
+  return !!matches;
+}
+
+function isAllValuesRegexOrNone(values) {
+  if (!values || !Array.isArray(values) || values.length === 0) {
+    return true;
+  }
+
+  const firstValuesIsRegex = isStartsWithRegex(values[0].$regex);
+  if (values.length === 1) {
+    return firstValuesIsRegex;
+  }
+
+  for (let i = 1, length = values.length; i < length; ++i) {
+    if (firstValuesIsRegex !== isStartsWithRegex(values[i].$regex)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function isAnyValueRegexStartsWith(values) {
+  return values.some(function (value) {
+    return isStartsWithRegex(value.$regex);
+  });
+}
+
 function createLiteralRegex(remaining) {
   return remaining.split('').map(c => {
     if (c.match(/[0-9a-zA-Z]/) !== null) {

src/Adapters/Storage/Postgres/sql/array/contains-all-regex.sql

@@ -0,0 +1,14 @@
+CREATE OR REPLACE FUNCTION array_contains_all_regex(
+  "array"   jsonb,
+  "values"  jsonb
+)
+  RETURNS boolean
+  LANGUAGE sql
+  IMMUTABLE
+  STRICT
+AS $function$
+  SELECT CASE
+	WHEN 0 = jsonb_array_length("values") THEN true = false
+	ELSE (SELECT RES.CNT = jsonb_array_length("values") FROM (SELECT COUNT(*) as CNT FROM jsonb_array_elements_text("array") as elt WHERE elt LIKE ANY (SELECT jsonb_array_elements_text("values"))) as RES)
+  END;
+$function$;

src/Adapters/Storage/Postgres/sql/array/contains-all.sql

@@ -7,5 +7,8 @@ CREATE OR REPLACE FUNCTION array_contains_all(
   IMMUTABLE
   STRICT
 AS $function$
-  SELECT RES.CNT = jsonb_array_length("values") FROM (SELECT COUNT(*) as CNT FROM jsonb_array_elements("array") as elt WHERE elt IN (SELECT jsonb_array_elements("values"))) as RES;
+  SELECT CASE
+	WHEN 0 = jsonb_array_length("values") THEN true = false
+	ELSE (SELECT RES.CNT = jsonb_array_length("values") FROM (SELECT COUNT(*) as CNT FROM jsonb_array_elements_text("array") as elt WHERE elt IN (SELECT jsonb_array_elements_text("values"))) as RES)
+  END;
 $function$;

src/Adapters/Storage/Postgres/sql/index.js

@@ -9,6 +9,7 @@ module.exports = {
     addUnique: sql('array/add-unique.sql'),
     contains: sql('array/contains.sql'),
     containsAll: sql('array/contains-all.sql'),
+    containsAllRegex: sql('array/contains-all-regex.sql'),
     remove: sql('array/remove.sql')
   },
   misc: {