Rolling Pool Update incomplete
-
Over that last two patch bundles my Rolling Pool Update fails to complete,
VMs are evacuated from the pool master to the pool slave (a 2 node pool).
Patches are applied to the pool master.
The pool master reboots.
After I can see that the pool master console is up, I reconnect XOA to the master.
I wait, wait, and wait some more and nothing else happens after that. The VMs remain on the slave server.
The "Rolling pool update" task is still in "Started" state.
There is an "API call: vm.stats" task that started after that in a failed state.{ "id": "0m1ku3cog", "properties": { "method": "vm.stats", "params": { "id": "ad5850fb-8264-18e2-c974-9df9ccaa6ccc" }, "name": "API call: vm.stats", "userId": "2844af20-1bee-43f8-9b91-e1ac3b49239f", "type": "api.call" }, "start": 1727448260848, "status": "failure", "updatedAt": 1727448260937, "end": 1727448260937, "result": { "message": "unusable", "name": "TypeError", "stack": "TypeError: unusable\n at /opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/undici/lib/api/readable.js:224:34\n at Promise._execute (/opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/bluebird/js/release/debuggability.js:384:9)\n at Promise._resolveFromExecutor (/opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/bluebird/js/release/promise.js:518:18)\n at new Promise (/opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/bluebird/js/release/promise.js:103:10)\n at consume (/opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/undici/lib/api/readable.js:212:10)\n at BodyReadable.text (/opt/xo/xo-builds/xen-orchestra-202409271433/node_modules/undici/lib/api/readable.js:111:12)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi-stats.mjs:258:39\n at XapiStats._getAndUpdateStats (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi-stats.mjs:319:18)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Api.#callApiMethod (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/api.mjs:402:20)" } }
The "Rolling pool update" task finally timed out.
{ "id": "0m1ku1muz", "properties": { "poolId": "ac9585b6-39ac-016c-f864-8c75b00c082b", "poolName": "Gen9", "progress": 40, "name": "Rolling pool update", "userId": "2844af20-1bee-43f8-9b91-e1ac3b49239f" }, "start": 1727448180731, "status": "failure", "updatedAt": 1727449839025, "tasks": [ { "id": "y5cpp9ox6vg", "properties": { "name": "Listing missing patches", "total": 2, "progress": 100 }, "start": 1727448180736, "status": "success", "tasks": [ { "id": "x7rtdxs48rl", "properties": { "name": "Listing missing patches for host 0366c500-c154-4967-8f12-fc45cf9390a5", "hostId": "0366c500-c154-4967-8f12-fc45cf9390a5", "hostName": "xcpng02" }, "start": 1727448180737, "status": "success", "end": 1727448180738 }, { "id": "11re7s9vjpkb", "properties": { "name": "Listing missing patches for host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448180737, "status": "success", "end": 1727448180738 } ], "end": 1727448180738 }, { "id": "pq07ubwpo", "properties": { "name": "Updating and rebooting" }, "start": 1727448180738, "status": "failure", "tasks": [ { "id": "kut3i9gog4m", "properties": { "name": "Restarting hosts", "progress": 33 }, "start": 1727448180824, "status": "failure", "tasks": [ { "id": "54axt2hik3c", "properties": { "name": "Restarting host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448180824, "status": "failure", "tasks": [ { "id": "ocube7c6kmf", "properties": { "name": "Evacuate", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448181014, "status": "success", "end": 1727448592236 }, { "id": "millse8k12o", "properties": { "name": "Installing patches", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448592237, "status": "success", "end": 1727448638798 }, { "id": "5aazbc573dg", "properties": { "name": "Restart", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448638799, "status": "success", "end": 1727448638986 }, { "id": "1roala4dv69", "properties": { "name": "Waiting for host to be up", "hostId": "7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e", "hostName": "xcpng01" }, "start": 1727448638986, "status": "failure", "end": 1727449839025, "result": { "message": "Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart", "name": "Error", "stack": "Error: Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:152:17\n at /opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:40\n at AsyncLocalStorage.run (node:async_hooks:346:14)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:41)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:31)\n at Function.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:27)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:142:24\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:112:11\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Xapi.rollingPoolReboot (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:102:5)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/patching.mjs:524:7\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at XenServers.rollingPoolUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/xen-servers.mjs:703:5)\n at Xo.rollingUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/api/pool.mjs:243:3)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)" } } ], "end": 1727449839025, "result": { "message": "Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart", "name": "Error", "stack": "Error: Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:152:17\n at /opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:40\n at AsyncLocalStorage.run (node:async_hooks:346:14)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:41)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:31)\n at Function.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:27)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:142:24\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:112:11\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Xapi.rollingPoolReboot (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:102:5)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/patching.mjs:524:7\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at XenServers.rollingPoolUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/xen-servers.mjs:703:5)\n at Xo.rollingUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/api/pool.mjs:243:3)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)" } } ], "end": 1727449839025, "result": { "message": "Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart", "name": "Error", "stack": "Error: Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:152:17\n at /opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:40\n at AsyncLocalStorage.run (node:async_hooks:346:14)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:41)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:31)\n at Function.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:27)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:142:24\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:112:11\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Xapi.rollingPoolReboot (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:102:5)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/patching.mjs:524:7\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at XenServers.rollingPoolUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/xen-servers.mjs:703:5)\n at Xo.rollingUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/api/pool.mjs:243:3)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)" } } ], "end": 1727449839025, "result": { "message": "Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart", "name": "Error", "stack": "Error: Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:152:17\n at /opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:40\n at AsyncLocalStorage.run (node:async_hooks:346:14)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:41)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:31)\n at Function.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:27)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:142:24\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:112:11\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Xapi.rollingPoolReboot (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:102:5)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/patching.mjs:524:7\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at XenServers.rollingPoolUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/xen-servers.mjs:703:5)\n at Xo.rollingUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/api/pool.mjs:243:3)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)" } } ], "end": 1727449839025, "result": { "message": "Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart", "name": "Error", "stack": "Error: Host 7cbc09aa-6d32-44a9-b6a2-eb5b30c11e1e took too long to restart\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:152:17\n at /opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:40\n at AsyncLocalStorage.run (node:async_hooks:346:14)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:41)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:31)\n at Function.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:54:27)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:142:24\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:112:11\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Xapi.rollingPoolReboot (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/pool.mjs:102:5)\n at file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xapi/mixins/patching.mjs:524:7\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)\n at XenServers.rollingPoolUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/xo-mixins/xen-servers.mjs:703:5)\n at Xo.rollingUpdate (file:///opt/xo/xo-builds/xen-orchestra-202409271433/packages/xo-server/src/api/pool.mjs:243:3)\n at Task.runInside (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:169:22)\n at Task.run (/opt/xo/xo-builds/xen-orchestra-202409271433/@vates/task/index.js:153:20)" } }
The way I have mitigated it in the past was to vacate the slave node and update the patches on it.
Then I have to go into the pool master and cleanse the failed/stale tasks.It's a bloody mess.
Any advice is welcome.
Danny -
Perhaps this was because there were 3 pages of failed tasks?
I have deleted them all with xo-cli and I'll see how the next patch cycle goes. -
@julien-f is aware of this problem and we'll find a way to make it more obvious
-