Backup failure: HTTP request has been canceled
-
I got an error during this weeks weekly full backup. It failed to backup to both backup servers. This is the first time it happes, although I had a similar error a while ago when it failed on one of the backup servers.
ERRORS:
HTTP request has been canceled - all targets have failed, step: writer.run()I updated xcp-ng on the 8th of July after the previous error. XOA is updated to the latest channel.
- node: 14.17.0 - npm: 6.14.13 - xen-orchestra-upload-ova: 0.1.4 - xo-server: 5.80.0 - xo-server-auth-github-enterprise: 0.2.2 - xo-server-auth-google-enterprise: 0.2.2 - xo-server-auth-ldap-enterprise: 0.10.2 - xo-server-auth-saml-enterprise: 0.9.0 - xo-server-backup-reports-enterprise: 0.16.10 - xo-server-netbox-enterprise: 0.1.2 - xo-server-telemetry: 0.5.0 - xo-server-transport-email-enterprise: 0.6.0 - xo-server-transport-icinga2-enterprise: 0.1.1 - xo-server-transport-nagios-enterprise: 0.1.1 - xo-server-transport-slack-enterprise: 0.0.0 - xo-server-transport-xmpp-enterprise: 0.1.1 - xo-server-usage-report-enterprise: 0.10.0 - xo-server-xoa: 0.11.3 - xo-web-enterprise: 5.84.0 - xoa-cli: 0.26.1 - xoa-updater: 0.37.0
srv02{ "data": { "mode": "full", "reportWhen": "failure" }, "id": "1626038700001", "jobId": "7d984eab-b12f-4d8a-81b6-2588e7ca6cb6", "jobName": "vm2 - Weekly full backup", "message": "backup", "scheduleId": "14b8c81f-3308-4089-a394-761f1b09b009", "start": 1626038700001, "status": "failure", "infos": [ { "data": { "vms": [ "2cfd595d-9fbc-dbaf-8965-08152370e5ff" ] }, "message": "vms" } ], "tasks": [ { "data": { "type": "VM", "id": "2cfd595d-9fbc-dbaf-8965-08152370e5ff" }, "id": "1626038703150:0", "message": "backup VM", "start": 1626038703150, "status": "failure", "tasks": [ { "id": "1626038703482", "message": "snapshot", "start": 1626038703482, "status": "success", "end": 1626038714904, "result": "05b62691-612e-da89-2fce-0e85265f2eea" }, { "data": { "id": "21d9217a-4764-4c3f-8580-95fd4ff1b87c", "type": "remote", "isFull": true }, "id": "1626038714913", "message": "export", "start": 1626038714913, "status": "failure", "tasks": [ { "id": "1626038714947", "message": "transfer", "start": 1626038714947, "status": "failure", "end": 1626041864738, "result": { "canceled": true, "method": "GET", "url": "http://srv02.mydomain.net/export/?ref=OpaqueRef%3A6656b1a9-fccd-44da-9a5b-7280fa38039a&use_compression=zstd&session_id=OpaqueRef%3A1bd7086f-9de3-4a4d-a5e6-e89b425ec27e&task_id=OpaqueRef%3A931b4337-5235-413b-bf9b-99e53810f0c9", "message": "HTTP request has been canceled", "name": "Error", "stack": "Error: HTTP request has been canceled\n at IncomingMessage.emitAbortedError (/usr/local/lib/node_modules/xo-server/node_modules/http-request-plus/index.js:79:19)\n at Object.onceWrapper (events.js:482:28)\n at IncomingMessage.emit (events.js:376:20)\n at IncomingMessage.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at IncomingMessage.emit (domain.js:470:12)\n at Socket.socketCloseListener (_http_client.js:432:11)\n at Socket.emit (events.js:388:22)\n at Socket.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at Socket.emit (domain.js:470:12)\n at TCP.<anonymous> (net.js:673:12)" } } ], "end": 1626041864739, "result": { "canceled": true, "method": "GET", "url": "http://srv02.mydomain.net/export/?ref=OpaqueRef%3A6656b1a9-fccd-44da-9a5b-7280fa38039a&use_compression=zstd&session_id=OpaqueRef%3A1bd7086f-9de3-4a4d-a5e6-e89b425ec27e&task_id=OpaqueRef%3A931b4337-5235-413b-bf9b-99e53810f0c9", "message": "HTTP request has been canceled", "name": "Error", "stack": "Error: HTTP request has been canceled\n at IncomingMessage.emitAbortedError (/usr/local/lib/node_modules/xo-server/node_modules/http-request-plus/index.js:79:19)\n at Object.onceWrapper (events.js:482:28)\n at IncomingMessage.emit (events.js:376:20)\n at IncomingMessage.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at IncomingMessage.emit (domain.js:470:12)\n at Socket.socketCloseListener (_http_client.js:432:11)\n at Socket.emit (events.js:388:22)\n at Socket.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at Socket.emit (domain.js:470:12)\n at TCP.<anonymous> (net.js:673:12)" } }, { "data": { "id": "c0ccbfd7-21fd-4d99-8bc5-9a8ac89d47f7", "type": "remote", "isFull": true }, "id": "1626038714914", "message": "export", "start": 1626038714914, "status": "failure", "tasks": [ { "id": "1626038715204", "message": "transfer", "start": 1626038715204, "status": "failure", "end": 1626041864964, "result": { "canceled": true, "method": "GET", "url": "http://srv02.mydomain.net/export/?ref=OpaqueRef%3A6656b1a9-fccd-44da-9a5b-7280fa38039a&use_compression=zstd&session_id=OpaqueRef%3A1bd7086f-9de3-4a4d-a5e6-e89b425ec27e&task_id=OpaqueRef%3A931b4337-5235-413b-bf9b-99e53810f0c9", "message": "HTTP request has been canceled", "name": "Error", "stack": "Error: HTTP request has been canceled\n at IncomingMessage.emitAbortedError (/usr/local/lib/node_modules/xo-server/node_modules/http-request-plus/index.js:79:19)\n at Object.onceWrapper (events.js:482:28)\n at IncomingMessage.emit (events.js:376:20)\n at IncomingMessage.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at IncomingMessage.emit (domain.js:470:12)\n at Socket.socketCloseListener (_http_client.js:432:11)\n at Socket.emit (events.js:388:22)\n at Socket.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at Socket.emit (domain.js:470:12)\n at TCP.<anonymous> (net.js:673:12)" } } ], "end": 1626041864964, "result": { "canceled": true, "method": "GET", "url": "http://srv02.mydomain.net/export/?ref=OpaqueRef%3A6656b1a9-fccd-44da-9a5b-7280fa38039a&use_compression=zstd&session_id=OpaqueRef%3A1bd7086f-9de3-4a4d-a5e6-e89b425ec27e&task_id=OpaqueRef%3A931b4337-5235-413b-bf9b-99e53810f0c9", "message": "HTTP request has been canceled", "name": "Error", "stack": "Error: HTTP request has been canceled\n at IncomingMessage.emitAbortedError (/usr/local/lib/node_modules/xo-server/node_modules/http-request-plus/index.js:79:19)\n at Object.onceWrapper (events.js:482:28)\n at IncomingMessage.emit (events.js:376:20)\n at IncomingMessage.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at IncomingMessage.emit (domain.js:470:12)\n at Socket.socketCloseListener (_http_client.js:432:11)\n at Socket.emit (events.js:388:22)\n at Socket.patchedEmit (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/log/configure.js:93:17)\n at Socket.emit (domain.js:470:12)\n at TCP.<anonymous> (net.js:673:12)" } } ], "end": 1626042095802, "result": { "message": "all targets have failed, step: writer.run()", "name": "Error", "stack": "Error: all targets have failed, step: writer.run()\n at VmBackup._callWriters (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/backups/_VmBackup.js:130:13)\n at async VmBackup._copyFull (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/backups/_VmBackup.js:248:5)\n at async VmBackup.run (/usr/local/lib/node_modules/xo-server/node_modules/@xen-orchestra/backups/_VmBackup.js:383:9)" } } ], "end": 1626042095802 }The previous error was for an incremental backup.
-
Hi,
If you are using XOA, it's preferable to open a support ticket so we can access it remotely and try to understand what's going on.
EAI_AGAINsounds like a DNS issue though. -
@olivierlambert I've opened a ticket.
The EAI_AGAIN I don't think it is a DNS issue. Were using fixed IPs only. I've seen it before but could not understand where it came from. Since then I've added all servers to their respective hosts-files, but that shouldn' be needed.
-
EAI_AGAIN is a DNS lookup timed out error, means it is a network connectivity error or proxy related error.
It's a problem when XO tries to get address from your
srv02.xxx.com. -
@olivierlambert said in Backup failure: HTTP request has been canceled:
It's a problem when XO tries to get address from your srv02.xxx.com.
It just makes no sense that DNS would be an issue on this network. I guess if XO momentarilly looses connection during DNS lookup it could cause this problem. I've noticed sometimes that the webui of XOA sometimes can show red ? if I start live several live migrations. Maybe unrelated?
