I've set up a fairly long mongo aggregate query to join several mongo collections together and shape up them into output of set of string fields. The query works fine as long as all the required values (ie : ids) exists but it breaks when it encounters null or empty values when doing the $lookup.

Following is the patientFile collection thats being queried :

 {
    "no" : "2020921008981",
    "startDateTime" : ISODate("2020-04-01T05:19:02.263+0000")
    "saleId" : "5e8424464475140d19c6941b",
    "patientId" : "5e8424464475140d1955941b"
 }

sale collection :

 {
    "_id" : ObjectId("5e8424464475140d19c6941b"),
    "invoices" : [
        {
          "billNumber" : "2020921053467", 
          "type" : "CREDIT",
          "insurancePlanId" : "160"
        },
        {
          "billNumber" : "2020921053469", 
          "type" : "DEBIT",
          "insurancePlanId" : "161"
        }
    ],
     "status" : "COMPLETE"
 }

insurance collection :

  { 
    "_id" : ObjectId("5b55aca20550de00210a6d25"), 
    "name" : "HIJKL" 
    "plans" : [
       {
         "_id" : "160", 
         "name" : "UVWZ", 
       }, 
       { 
         "_id" : "161", 
         "name" : "LMNO", 
       }
     ]
  }

patient collection :

{ 
  "_id" : ObjectId("5b55cc5c0550de00217ae0f3"),  
  "name" : "TAN NAI",
  "userId" : {
    "number" : "787333128H"
    }
}

Heres the aggregate query :

  db.getCollection("patientFile").aggregate([
  { $match: { "startDateTime": { $gte: ISODate("2020-01-01T00:00:00.000Z"), 
                          $lt: ISODate("2020-05-01T00:00:00.000Z") } } },

  {
    $lookup:
        {
            from: "patient",
            let: { pid: "$patientId" },
            pipeline: [
                {
                    $match: {
                        $expr: {
                            $eq: ["$_id", { $toObjectId: "$$pid" }]
                        }
                    }
                },
                { "$project": { "name": 1, "userId.number": 1, "_id": 0 } }
            ],
            as: "patient"
        }
   },
   {
    $lookup:
        {
            from: "sale",
            let: { sid: "$saleId" },
            pipeline: [
                {
                    $match: {
                        $expr: {
                            $eq: ["$_id", { $toObjectId: "$$sid" }]
                        }
                    }
                }
            ],
            as: "sale"
        }
},

{ $unwind: "$sale" },
{ $unwind: "$patient" },

{
    $lookup: {
        from: "insurance",
        let: { pid: {$ifNull:["$sale.bill.insurancePlanId", [] ]} },
        pipeline: [
            {
                $unwind: "$plans"
            },
            {
                $match: { $expr: { $in: ["$plans._id", "$$pid"] } }
            },
            {
                $project: { _id: 0, name: 1 }
            }
        ],
        as: "insurances"
       }
  },

  { $match: { "insurances.name": { $exists: true, $ne: null } } },


  {
    $addFields: {

        invoice: {
            $reduce: {
                input:  {$ifNull:["$sale.bill.billNumber", [] ]},
                initialValue: "",
                in: {
                    $cond: [{ "$eq": ["$$value", ""] }, "$$this", { $concat: ["$$value", "\n", "$$this"] }]
                }
            }
        },
        insurances: {
            $reduce: {
                input:  {$ifNull:["$insurances.name", [] ]},
                initialValue: "",
                in: {
                    $cond: [{ "$eq": ["$$value", ""] }, "$$this", { $concat: ["$$value", "\n", "$$this"] }]
                }
             }
          }

       }
  },


  {
    "$project": {
        "startDateTime": 1,
        "patientName": "$patient.name",
        "invoice": 1,
        "insurances": 1
       }
     }

   ],

      { allowDiskUse: true }

 )

Error :

Unable to execute the selected commands

Mongo Server error (MongoCommandException): Command failed with error 241 (ConversionFailure): 'Failed to parse objectId '' in $convert with no onError value: Invalid string length for parsing to OID, expected 24 but found 0' on server localhost:27017. 

The full response is:
 { 
    "ok" : 0.0, 
    "errmsg" : "Failed to parse objectId '' in $convert with no onError value: Invalid string length for parsing to OID, expected 24 but found 0", 
    "code" : NumberInt(241), 
    "codeName" : "ConversionFailure"
 }

As a solution i have found, used $ifNull but this error keeps coming. What would be the best step to take for this scenario?

I see a couple of ways:

• Instead of converting the string value to an ObjectId to test, convert the ObjectId to a string

  $match: {
      $expr: {
          $eq: [{$toString: "$_id"}, "$$pid" ]
      }
  }
  

• Instead of the $toObjectId helper, use $convert and provide onError and/or onNull values:

  $match: {
      $expr: {
          $eq: ["$_id", { $convert: {
                                 input: "$$pid",
                                 to: "objectId", 
                                 onError: {error:true},
                                 onNull: {isnull:true}
          }}]
      }
  }