Last night just started getting the following errors.
When I first got error 2 out of 13 VMs failed. When I retired I am down to 1 vm failed. I retried again and same 1 vm fail again.
This is a mirror job to BackBlaze but not sure its related to BackBlaze.
{
"data": {
"mode": "delta",
"reportWhen": "always",
"backupReportTpl": "compactMjml",
"hideSuccessfulItems": true
},
"id": "1760959398810",
"jobId": "de8585b4-d801-44e6-a97a-331725a547c4",
"jobName": "Delilah - Mirror Backups Backblaze",
"message": "backup",
"scheduleId": "1e58ad4a-4b38-49a1-9dfc-4acfa3510681",
"start": 1760959398810,
"status": "failure",
"infos": [
{
"data": {
"vms": [
"0dc3c884-97bb-bcb5-62d1-50ddfebf1519",
"102a5ac2-ab7f-68ca-9fa1-b549d64ec4a9",
"291f9d05-88c8-fc37-107a-368a55c649be",
"2c21c005-9230-f3f3-cdf6-2d39407b3456",
"342b470e-fac9-6579-f82d-2517b8de6112",
"4567b8da-3a3c-00a4-cf6b-ccdae302a645",
"5703adef-d804-6b15-ba2f-7b3357a711bb",
"5a2c18ca-e9ce-abfc-8bd7-2794f94c6c3a",
"a8858f86-ceab-2d5d-5d46-1b083c3fa399",
"a89ed090-610a-8f41-58b1-1cb1c8d6517a",
"ab9e8cde-fad8-5089-29c8-c904a810857a",
"b0c21701-7309-dd9c-fa2a-bdc84a24565e",
"bcc182c6-dbb7-33c4-3e8b-c7db9b246e7b",
"c9dadc90-6fb8-eee7-d3e5-e1ae1024a77f",
"d2025b59-6268-6bb9-28cd-e4812395563c",
"d4352870-d73c-5cda-9470-ffd7cd01e251",
"f96015ff-6a52-aec0-4581-4a65029437a7"
]
},
"message": "vms"
}
],
"tasks": [
{
"data": {
"type": "VM",
"id": "102a5ac2-ab7f-68ca-9fa1-b549d64ec4a9"
},
"id": "1760959399518",
"message": "backup VM",
"start": 1760959399518,
"status": "success",
"tasks": [
{
"id": "1760959399540",
"message": "clean-vm",
"start": 1760959399540,
"status": "success",
"end": 1760959448830,
"result": {
"merge": false
}
},
{
"id": "1760959448856:1",
"message": "clean-vm",
"start": 1760959448856,
"status": "success",
"end": 1760959493080,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959493080
},
{
"data": {
"type": "VM",
"id": "291f9d05-88c8-fc37-107a-368a55c649be"
},
"id": "1760959399538",
"message": "backup VM",
"start": 1760959399538,
"status": "success",
"tasks": [
{
"id": "1760959399660",
"message": "clean-vm",
"start": 1760959399660,
"status": "success",
"end": 1760959435933,
"result": {
"merge": false
}
},
{
"id": "1760959436036",
"message": "clean-vm",
"start": 1760959436036,
"status": "success",
"end": 1760959473882,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959473882
},
{
"data": {
"type": "VM",
"id": "2c21c005-9230-f3f3-cdf6-2d39407b3456"
},
"id": "1760959473911",
"message": "backup VM",
"start": 1760959473911,
"status": "success",
"tasks": [
{
"id": "1760959473929",
"message": "clean-vm",
"start": 1760959473929,
"status": "success",
"end": 1760959509274,
"result": {
"merge": false
}
},
{
"id": "1760959509392",
"message": "clean-vm",
"start": 1760959509392,
"status": "success",
"end": 1760959545428,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959545428
},
{
"data": {
"type": "VM",
"id": "342b470e-fac9-6579-f82d-2517b8de6112"
},
"id": "1760959493083",
"message": "backup VM",
"start": 1760959493083,
"status": "success",
"tasks": [
{
"id": "1760959493100",
"message": "clean-vm",
"start": 1760959493100,
"status": "success",
"end": 1760959516729,
"result": {
"merge": false
}
},
{
"id": "1760959516758",
"message": "clean-vm",
"start": 1760959516758,
"status": "success",
"end": 1760959541506,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959541506
},
{
"data": {
"type": "VM",
"id": "4567b8da-3a3c-00a4-cf6b-ccdae302a645"
},
"id": "1760959541521",
"message": "backup VM",
"start": 1760959541521,
"status": "success",
"tasks": [
{
"id": "1760959541558",
"message": "clean-vm",
"start": 1760959541558,
"status": "success",
"end": 1760959549245,
"result": {
"merge": false
}
},
{
"id": "1760959549340",
"message": "clean-vm",
"start": 1760959549340,
"status": "success",
"end": 1760959555142,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959555142
},
{
"data": {
"type": "VM",
"id": "5703adef-d804-6b15-ba2f-7b3357a711bb"
},
"id": "1760959545440",
"message": "backup VM",
"start": 1760959545440,
"status": "success",
"tasks": [
{
"id": "1760959545457",
"message": "clean-vm",
"start": 1760959545457,
"status": "success",
"end": 1760959545948,
"result": {
"merge": false
}
},
{
"id": "1760959545981:1",
"message": "clean-vm",
"start": 1760959545981,
"status": "success",
"end": 1760959546221,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "This VM is excluded by the job filter"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959546222
},
{
"data": {
"type": "VM",
"id": "5a2c18ca-e9ce-abfc-8bd7-2794f94c6c3a"
},
"id": "1760959546230",
"message": "backup VM",
"start": 1760959546230,
"status": "success",
"tasks": [
{
"id": "1760959546246",
"message": "clean-vm",
"start": 1760959546246,
"status": "success",
"end": 1760959564273,
"result": {
"merge": false
}
},
{
"id": "1760959564357:1",
"message": "clean-vm",
"start": 1760959564357,
"status": "success",
"end": 1760959578416,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "This VM is excluded by the job filter"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959578416
},
{
"data": {
"type": "VM",
"id": "a8858f86-ceab-2d5d-5d46-1b083c3fa399"
},
"id": "1760959555146",
"message": "backup VM",
"start": 1760959555146,
"status": "success",
"tasks": [
{
"id": "1760959555178",
"message": "clean-vm",
"start": 1760959555178,
"status": "success",
"end": 1760959629767,
"result": {
"merge": false
}
},
{
"id": "1760959629795",
"message": "clean-vm",
"start": 1760959629795,
"status": "success",
"end": 1760959692700,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959692700
},
{
"data": {
"type": "VM",
"id": "ab9e8cde-fad8-5089-29c8-c904a810857a"
},
"id": "1760959578446",
"message": "backup VM",
"start": 1760959578446,
"status": "success",
"tasks": [
{
"id": "1760959578466",
"message": "clean-vm",
"start": 1760959578466,
"status": "success",
"end": 1760959632497,
"result": {
"merge": false
}
},
{
"id": "1760959632590:0",
"message": "clean-vm",
"start": 1760959632590,
"status": "success",
"end": 1760959696721,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959696721
},
{
"data": {
"type": "VM",
"id": "b0c21701-7309-dd9c-fa2a-bdc84a24565e"
},
"id": "1760959692705",
"message": "backup VM",
"start": 1760959692705,
"status": "success",
"tasks": [
{
"id": "1760959692720",
"message": "clean-vm",
"start": 1760959692720,
"status": "success",
"end": 1760959743473,
"result": {
"merge": false
}
},
{
"id": "1760959743495:1",
"message": "clean-vm",
"start": 1760959743495,
"status": "success",
"end": 1760959795554,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "This VM is excluded by the job filter"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959795554
},
{
"data": {
"type": "VM",
"id": "bcc182c6-dbb7-33c4-3e8b-c7db9b246e7b"
},
"id": "1760959696739",
"message": "backup VM",
"start": 1760959696739,
"status": "success",
"tasks": [
{
"id": "1760959696765",
"message": "clean-vm",
"start": 1760959696765,
"status": "success",
"end": 1760959726538,
"result": {
"merge": false
}
},
{
"id": "1760959726636:0",
"message": "clean-vm",
"start": 1760959726636,
"status": "success",
"end": 1760959754694,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959754694
},
{
"data": {
"type": "VM",
"id": "c9dadc90-6fb8-eee7-d3e5-e1ae1024a77f"
},
"id": "1760959754698",
"message": "backup VM",
"start": 1760959754698,
"status": "failure",
"tasks": [
{
"id": "1760959754715",
"message": "clean-vm",
"start": 1760959754715,
"status": "failure",
"tasks": [
{
"id": "1760959795421",
"message": "merge",
"start": 1760959795421,
"status": "failure",
"end": 1760959824455,
"result": {
"name": "InvalidRequest",
"$fault": "client",
"$metadata": {
"httpStatusCode": 400,
"requestId": "4f8fb6bb5ee77752",
"extendedRequestId": "aYvU1qDBHYcE2jmTNOBg5bGUZZSo0UTA/",
"attempts": 1,
"totalRetryDelay": 0
},
"Code": "InvalidRequest",
"message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532",
"stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22"
}
}
],
"end": 1760959824455,
"result": {
"name": "InvalidRequest",
"$fault": "client",
"$metadata": {
"httpStatusCode": 400,
"requestId": "4f8fb6bb5ee77752",
"extendedRequestId": "aYvU1qDBHYcE2jmTNOBg5bGUZZSo0UTA/",
"attempts": 1,
"totalRetryDelay": 0
},
"Code": "InvalidRequest",
"message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532",
"stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22"
}
},
{
"id": "1760959824478:1",
"message": "clean-vm",
"start": 1760959824478,
"status": "failure",
"tasks": [
{
"id": "1760959861600",
"message": "merge",
"start": 1760959861600,
"status": "failure",
"end": 1760959889737,
"result": {
"name": "InvalidRequest",
"$fault": "client",
"$metadata": {
"httpStatusCode": 400,
"requestId": "130a17c8cff103db",
"extendedRequestId": "aYpo1YjBJYQo2X2RMOP85oWUXZUo09DDB",
"attempts": 1,
"totalRetryDelay": 0
},
"Code": "InvalidRequest",
"message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532",
"stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22"
}
}
],
"end": 1760959889737,
"result": {
"name": "InvalidRequest",
"$fault": "client",
"$metadata": {
"httpStatusCode": 400,
"requestId": "130a17c8cff103db",
"extendedRequestId": "aYpo1YjBJYQo2X2RMOP85oWUXZUo09DDB",
"attempts": 1,
"totalRetryDelay": 0
},
"Code": "InvalidRequest",
"message": "no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532",
"stack": "InvalidRequest: no such file: 4_zfb35008ab69d78799e8e0410_f412dd3db97ba5166_d20251020_m052045_c005_v7007000_t0000_u01760937645532\n at throwDefaultError (/usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:836:20)\n at /usr/local/lib/node_modules/xo-server/node_modules/@smithy/smithy-client/dist-cjs/index.js:845:5\n at de_CommandError (/usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/client-s3/dist-cjs/index.js:4739:14)\n at process.processTicksAndRejections (node:internal/process/task_queues:95:5)\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-serde/dist-cjs/index.js:35:20\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:485:18\n at async /usr/local/lib/node_modules/xo-server/node_modules/@smithy/middleware-retry/dist-cjs/index.js:320:38\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:110:22\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/index.js:138:14\n at async /usr/local/lib/node_modules/xo-server/node_modules/@aws-sdk/middleware-logger/dist-cjs/index.js:34:22"
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959889738
},
{
"data": {
"type": "VM",
"id": "d4352870-d73c-5cda-9470-ffd7cd01e251"
},
"id": "1760959795559",
"message": "backup VM",
"start": 1760959795559,
"status": "success",
"tasks": [
{
"id": "1760959795574",
"message": "clean-vm",
"start": 1760959795574,
"status": "success",
"end": 1760959804514,
"result": {
"merge": false
}
},
{
"id": "1760959804585",
"message": "clean-vm",
"start": 1760959804585,
"status": "success",
"end": 1760959810828,
"result": {
"merge": false
}
}
],
"infos": [
{
"message": "No new data to upload for this VM"
},
{
"message": "No healthCheck needed because no data was transferred."
}
],
"end": 1760959810828
}
],
"end": 1760959889738
}
Edit - posted correct error log.
