@poddingue Hi!
Unfortunately, another error came up on the larger vms. The job also seems to be painfully slow. Slower than usual I mean.
Also, it seems that job changes are not being commited. I've changed the job so it wouldn't retry in case of failure and the log shows it still trying 3 times (that was the old setting, and probably explains the job taking longer than usual).
[image: 1784017759119-a917295e-ea92-4173-bc5a-7c9652a8d2ad-image.jpeg]
{
"id": "0mrjahr8e-ssawlbveq5",
"start": 1783951215134,
"status": "failure",
"tasks": [
{
"id": "0mrjahs3e-dbfaqajeze",
"start": 1783951216250,
"status": "failure",
"tasks": [
{
"id": "0mrjahs8s-2yb6i3fny6f",
"start": 1783951216444,
"status": "failure",
"end": 1783959266484,
"result": {
"$fault": "server",
"$metadata": {
"httpStatusCode": 500,
"requestId": "c0a9f30190109f99",
"extendedRequestId": "aMJgyljkPZEdiEDW0Nk9lTmbINpI1FGIP",
"attempts": 3,
"totalRetryDelay": 197
},
"name": "InternalError",
"Code": "InternalError",
"message": "internal incident",
"stack": "InternalError: internal incident\n at ProtocolLib.getErrorSchemaOrThrowBaseException (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:70:67)\n at S3RestXmlProtocol.handleError (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:1856:65)\n at S3RestXmlProtocol.deserializeResponse (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/protocols/index.js:341:24)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/schema/index.js:25:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:367:20\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/retry/index.js:172:50\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/checksums/dist-cjs/index.js:337:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:64:28\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:91:20"
},
"message": "transfer",
"data": {
"progress": 40
}
}
],
"end": 1783959266485,
"result": {
"$fault": "server",
"$metadata": {
"httpStatusCode": 500,
"requestId": "c0a9f30190109f99",
"extendedRequestId": "aMJgyljkPZEdiEDW0Nk9lTmbINpI1FGIP",
"attempts": 3,
"totalRetryDelay": 197
},
"name": "InternalError",
"Code": "InternalError",
"message": "internal incident",
"stack": "InternalError: internal incident\n at ProtocolLib.getErrorSchemaOrThrowBaseException (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:70:67)\n at S3RestXmlProtocol.handleError (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:1856:65)\n at S3RestXmlProtocol.deserializeResponse (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/protocols/index.js:341:24)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/schema/index.js:25:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:367:20\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/retry/index.js:172:50\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/checksums/dist-cjs/index.js:337:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:64:28\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:91:20"
},
"message": "export",
"data": {
"id": "14b59f44-4517-411e-9e39-4eb7136928d0",
"type": "remote",
"isFull": true
}
},
{
"id": "0mrjfabp3-c8mf2pnq8yf",
"start": 1783959266487,
"status": "success",
"warnings": [
{
"data": {
"path": "xo-vm-backups/60b4d2a7-3f91-f0f9-b0a0-f834df3ff9ee/cache.json.gz",
"actual": 0,
"expected": 1
},
"message": "unexpected number of entries in backup cache"
}
],
"end": 1783959266989,
"result": {
"merge": false,
"size": 0
},
"message": "clean-vm"
}
],
"end": 1783959266989,
"result": {
"$fault": "server",
"$metadata": {
"httpStatusCode": 500,
"requestId": "c0a9f30190109f99",
"extendedRequestId": "aMJgyljkPZEdiEDW0Nk9lTmbINpI1FGIP",
"attempts": 3,
"totalRetryDelay": 197
},
"name": "InternalError",
"Code": "InternalError",
"message": "internal incident",
"stack": "InternalError: internal incident\n at ProtocolLib.getErrorSchemaOrThrowBaseException (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:70:67)\n at S3RestXmlProtocol.handleError (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/core/dist-cjs/submodules/protocols/index.js:1856:65)\n at S3RestXmlProtocol.deserializeResponse (/opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/protocols/index.js:341:24)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/schema/index.js:25:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:367:20\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@smithy/core/dist-cjs/submodules/retry/index.js:172:50\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/checksums/dist-cjs/index.js:337:24\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:64:28\n at async /opt/xo/xo-builds/xen-orchestra-202607120728/node_modules/@aws-sdk/middleware-sdk-s3/dist-cjs/submodules/s3/index.js:91:20"
},
"message": "backup VM",
"data": {
"id": "60b4d2a7-3f91-f0f9-b0a0-f834df3ff9ee",
"type": "VM",
"progress": 0
}
},