jstests/noPassthrough/change_streams_pre_image_removal_job.js
// 'batchedDelete' determines whether pre-images will be removed in batches or document-by-document.
'batchedDelete' determines whether pre-images will be removed in batches or document-by-document.
' batchedDelete ' determines whether pre - images will be removed in batches or document - by - document .
function testPreImageRemovalJob(batchedDelete) {
    let lastOplogEntryToBeRemoved = getLatestOp(primaryNode);
    while (!oplogIsRolledOver(lastOplogEntryToBeRemoved.ts)) {
        assert.commandWorked(
            testDB.tmp.insert({long_str: largeStr}, {writeConcern: {w: "majority"}}));
    }
    assert.soon(() => getPreImages(primaryNode).length == 0);
    assert.commandWorked(primaryNode.adminCommand(
        {setParameter: 1, batchedExpiredChangeStreamPreImageRemoval: batchedDelete}));
    const collA = assertDropAndRecreateCollection(
        testDB, "collA", {changeStreamPreAndPostImages: {enabled: true}});
    const collB = assertDropAndRecreateCollection(
        testDB, "collB", {changeStreamPreAndPostImages: {enabled: true}});
    for (const coll of [collA, collB]) {
        assert.commandWorked(coll.insert(docA, {writeConcern: {w: "majority"}}));
        assert.commandWorked(coll.update(docA, {$inc: {version: 1}}));
        assert.commandWorked(coll.update(docB, {$inc: {version: 1}}));
    }
    let preImages = getPreImages(primaryNode);
    const preImagesToExpire = 4;
    assert.eq(preImages.length, preImagesToExpire, preImages);
    lastOplogEntryToBeRemoved = getLatestOp(primaryNode);
    assert.neq(lastOplogEntryToBeRemoved, null);
    while (!oplogIsRolledOver(lastOplogEntryToBeRemoved.ts)) {
        assert.commandWorked(collA.insert({long_str: largeStr}, {writeConcern: {w: "majority"}}));
    }
    for (const coll of [collA, collB]) {
        assert.commandWorked(coll.update(docC, {$inc: {version: 1}}));
    }
    assert.soon(() => {
        preImages = getPreImages(primaryNode);
        const onlyTwoPreImagesLeft = preImages.length == 2;
        const allPreImagesHaveBiggerTimestamp = preImages.every(
            preImage => timestampCmp(preImage._id.ts, lastOplogEntryToBeRemoved.ts) == 1);
        return onlyTwoPreImagesLeft && allPreImagesHaveBiggerTimestamp;
    });
    const preimagesNs = 'config.system.preimages';
    if (batchedDelete) {
         As pre-images span two collections, the minimum number of batches is 2, as we perform
         the range-deletion per collection. The maximum number of batches is 4 (one per single
         pre-image removed).
        const expectedNumberOfBatchesRange = [2, 3, 4];
        const serverStatusBatches = testDB.serverStatus()['batchedDeletes']['batches'];
        const serverStatusDocs = testDB.serverStatus()['batchedDeletes']['docs'];
        assert.contains(serverStatusBatches, expectedNumberOfBatchesRange);
        assert.eq(serverStatusDocs, preImagesToExpire);
        assert.contains(
            localDB.oplog.rs
                .find({ns: 'admin.$cmd', 'o.applyOps.op': 'd', 'o.applyOps.ns': preimagesNs})
                .itcount(),
            expectedNumberOfBatchesRange);
    } else {
        assert.eq(preImagesToExpire, localDB.oplog.rs.find({op: 'd', ns: preimagesNs}).itcount());
    }
    assert.eq(0, localDB.oplog.rs.find({op: {'$ne': 'd'}, ns: preimagesNs}).itcount());
     Verify that pre-images collection content on the primary node is the same as on the
     secondary.
    rst.awaitReplication();
    assert(bsonWoCompare(getPreImages(primaryNode), getPreImages(rst.getSecondary())) === 0);
}
function testPreImageRemovalJob ( batchedDelete ) { let lastOplogEntryToBeRemoved = getLatestOp ( primaryNode ) ; while ( ! oplogIsRolledOver ( lastOplogEntryToBeRemoved . ts ) ) { assert . commandWorked ( testDB . tmp . insert ( { long_str : largeStr } , { writeConcern : { w : "majority" } } ) ) ; } assert . soon ( ( ) => getPreImages ( primaryNode ) . length == 0 ) ; assert . commandWorked ( primaryNode . adminCommand ( { setParameter : 1 , batchedExpiredChangeStreamPreImageRemoval : batchedDelete } ) ) ; const collA = assertDropAndRecreateCollection ( testDB , "collA" , { changeStreamPreAndPostImages : { enabled : true } } ) ; const collB = assertDropAndRecreateCollection ( testDB , "collB" , { changeStreamPreAndPostImages : { enabled : true } } ) ; for ( const coll of [ collA , collB ] ) { assert . commandWorked ( coll . insert ( docA , { writeConcern : { w : "majority" } } ) ) ; assert . commandWorked ( coll . update ( docA , { $inc : { version : 1 } } ) ) ; assert . commandWorked ( coll . update ( docB , { $inc : { version : 1 } } ) ) ; } let preImages = getPreImages ( primaryNode ) ; const preImagesToExpire = 4 ; assert . eq ( preImages . length , preImagesToExpire , preImages ) ; lastOplogEntryToBeRemoved = getLatestOp ( primaryNode ) ; assert . neq ( lastOplogEntryToBeRemoved , null ) ; while ( ! oplogIsRolledOver ( lastOplogEntryToBeRemoved . ts ) ) { assert . commandWorked ( collA . insert ( { long_str : largeStr } , { writeConcern : { w : "majority" } } ) ) ; } for ( const coll of [ collA , collB ] ) { assert . commandWorked ( coll . update ( docC , { $inc : { version : 1 } } ) ) ; } assert . soon ( ( ) => { preImages = getPreImages ( primaryNode ) ; const onlyTwoPreImagesLeft = preImages . length == 2 ; const allPreImagesHaveBiggerTimestamp = preImages . every ( preImage => timestampCmp ( preImage . _id . ts , lastOplogEntryToBeRemoved . ts ) == 1 ) ; return onlyTwoPreImagesLeft && allPreImagesHaveBiggerTimestamp ; } ) ; const preimagesNs = 'config.system.preimages' ; if ( batchedDelete ) { const expectedNumberOfBatchesRange = [ 2 , 3 , 4 ] ; const serverStatusBatches = testDB . serverStatus ( ) [ 'batchedDeletes' ] [ 'batches' ] ; const serverStatusDocs = testDB . serverStatus ( ) [ 'batchedDeletes' ] [ 'docs' ] ; assert . contains ( serverStatusBatches , expectedNumberOfBatchesRange ) ; assert . eq ( serverStatusDocs , preImagesToExpire ) ; assert . contains ( localDB . oplog . rs . find ( { ns : 'admin.$cmd' , 'o.applyOps.op' : 'd' , 'o.applyOps.ns' : preimagesNs } ) . itcount ( ) , expectedNumberOfBatchesRange ) ; } else { assert . eq ( preImagesToExpire , localDB . oplog . rs . find ( { op : 'd' , ns : preimagesNs } ) . itcount ( ) ) ; } assert . eq ( 0 , localDB . oplog . rs . find ( { op : { '$ne' : 'd' } , ns : preimagesNs } ) . itcount ( ) ) ; rst . awaitReplication ( ) ; assert ( bsonWoCompare ( getPreImages ( primaryNode ) , getPreImages ( rst . getSecondary ( ) ) ) === 0 ) ; }
'batchedDelete' determines whether pre-images will be removed in batches or document-by-document.
' batchedDelete ' determines whether pre - images will be removed in batches or document - by - document .
// Roll over the oplog, leading to 'PeriodicChangeStreamExpiredPreImagesRemover' periodic job // deleting all pre-images. // Set the 'batchedExpiredChangeStreamPreImageRemoval'. // Drop and recreate the collections with pre-images recording. // Perform insert and update operations. // Pre-images collection should contain four pre-images. // Roll over all current oplog entries. // Checks if the oplog has been rolled over from the timestamp of // 'lastOplogEntryToBeRemoved', ie. the timestamp of the first entry in the oplog is greater // than the timestamp of the 'lastOplogEntryToBeRemoved' on each node of the replica set. // Perform update operations that insert new pre-images that are not expired yet. // Wait until 'PeriodicChangeStreamExpiredPreImagesRemover' periodic job will delete the expired // pre-images. // Only two pre-images should still be there, as their timestamp is greater than the oldest // oplog entry timestamp. // Because the pre-images collection is implicitly replicated, validate that writes do not // generate oplog entries, with the exception of deletions. // Multi-deletes are batched base on time before performing the deletion, therefore the // deleted pre-images can span through multiple applyOps oplog entries. // // As pre-images span two collections, the minimum number of batches is 2, as we perform // the range-deletion per collection. The maximum number of batches is 4 (one per single // pre-image removed). // Verify that pre-images collection content on the primary node is the same as on the // secondary.