S3 backup failure after update

[Andrew]

@julien-f
S3 delta backups fail after update..... (XO from source)
fail d9ead2d9f5cbcc642d3c03b7f62c8cd0ee4ddb1a
works 5b7228ed697697b243b34df985643b20a41d0faf

Error: Cannot read properties of undefined (reading 'call')

So somewhere in there causes a problem.

    {
      "data": {
        "type": "VM",
        "id": "00371171-0a0b-f32f-916e-a79e8f8a98d4"
      },
      "id": "1653455400938:1",
      "message": "backup VM",
      "start": 1653455400938,
      "status": "failure",
      "tasks": [
        {
          "id": "1653455448361",
          "message": "snapshot",
          "start": 1653455448361,
          "status": "success",
          "end": 1653455450792,
          "result": "8a724963-152c-fc6c-9032-efb327a30ad2"
        },
        {
          "data": {
            "id": "db9ad0a8-bce6-4a2b-b9fd-5c4cecf059c4",
            "isFull": false,
            "type": "remote"
          },
          "id": "1653455450792:0",
          "message": "export",
          "start": 1653455450792,
          "status": "failure",
          "tasks": [
            {
              "id": "1653455451549",
              "message": "transfer",
              "start": 1653455451549,
              "status": "success",
              "end": 1653455493043,
              "result": {
                "size": 463617024
              }
            },
            {
              "id": "1653455493817",
              "message": "health check",
              "start": 1653455493817,
              "status": "failure",
              "tasks": [
                {
                  "id": "1653455510268",
                  "message": "transfer",
                  "start": 1653455510268,
                  "status": "failure",
                  "end": 1653455510269,
                  "result": {
                    "message": "Cannot read properties of undefined (reading 'call')",
                    "name": "TypeError",
                    "stack": "TypeError: Cannot read properties of undefined (reading 'call')\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:48:34\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:108:24\n    at Zone.run (/opt/xo/xo-builds/xen-orchestra-202205241158/node_modules/node-zone/index.js:80:23)\n    at Task.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:106:23)\n    at Function.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:38:27)\n    at ImportVmBackup.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:42:17)\n    at runMicrotasks (<anonymous>)\n    at processTicksAndRejections (node:internal/process/task_queues:96:5)\n    at async /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/writers/DeltaBackupWriter.js:84:36"
                  }
                }
              ],
              "end": 1653455510269,
              "result": {
                "message": "Cannot read properties of undefined (reading 'call')",
                "name": "TypeError",
                "stack": "TypeError: Cannot read properties of undefined (reading 'call')\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:48:34\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:108:24\n    at Zone.run (/opt/xo/xo-builds/xen-orchestra-202205241158/node_modules/node-zone/index.js:80:23)\n    at Task.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:106:23)\n    at Function.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:38:27)\n    at ImportVmBackup.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:42:17)\n    at runMicrotasks (<anonymous>)\n    at processTicksAndRejections (node:internal/process/task_queues:96:5)\n    at async /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/writers/DeltaBackupWriter.js:84:36"
              }
            }
          ],
          "end": 1653455510269,
          "result": {
            "message": "Cannot read properties of undefined (reading 'call')",
            "name": "TypeError",
            "stack": "TypeError: Cannot read properties of undefined (reading 'call')\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:48:34\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:108:24\n    at Zone.run (/opt/xo/xo-builds/xen-orchestra-202205241158/node_modules/node-zone/index.js:80:23)\n    at Task.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:106:23)\n    at Function.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:38:27)\n    at ImportVmBackup.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:42:17)\n    at runMicrotasks (<anonymous>)\n    at processTicksAndRejections (node:internal/process/task_queues:96:5)\n    at async /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/writers/DeltaBackupWriter.js:84:36"
          }
        }
      ],
      "end": 1653455533845,
      "result": {
        "message": "Cannot read properties of undefined (reading 'call')",
        "name": "TypeError",
        "stack": "TypeError: Cannot read properties of undefined (reading 'call')\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:48:34\n    at /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:108:24\n    at Zone.run (/opt/xo/xo-builds/xen-orchestra-202205241158/node_modules/node-zone/index.js:80:23)\n    at Task.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:106:23)\n    at Function.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/Task.js:38:27)\n    at ImportVmBackup.run (/opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/ImportVmBackup.js:42:17)\n    at runMicrotasks (<anonymous>)\n    at processTicksAndRejections (node:internal/process/task_queues:96:5)\n    at async /opt/xo/xo-builds/xen-orchestra-202205241158/@xen-orchestra/backups/writers/DeltaBackupWriter.js:84:36"
      }
    },

[julien-f]

@Andrew @Danp It's merged, thank you

[olivierlambert]

Please try fix_health_check and report! Thanks

[Andrew]

@olivierlambert Yes, that works.

[Danp]

FWIW, I'm seeing the same issue for a non-S3 backup. Here is what the system log shows --

May 26 00:05:06 ubuntu-xo xo-server[11563]:   error: Error: task has already ended
May 26 00:05:06 ubuntu-xo xo-server[11563]:       at Task.logAfterEnd (/opt/xen-orchestra/@xen-orchestra/backups/Task.js:7:9)
May 26 00:05:06 ubuntu-xo xo-server[11563]:       at Task.#log (/opt/xen-orchestra/@xen-orchestra/backups/Task.js:144:16)
May 26 00:05:06 ubuntu-xo xo-server[11563]:       at Task.#end (/opt/xen-orchestra/@xen-orchestra/backups/Task.js:139:14)
May 26 00:05:06 ubuntu-xo xo-server[11563]:       at Task.success (/opt/xen-orchestra/@xen-orchestra/backups/Task.js:124:14)
May 26 00:05:06 ubuntu-xo xo-server[11563]:       at /opt/xen-orchestra/@xen-orchestra/backups/Task.js:111:52

Edit: Testing with fix_health_check and appears to work correctly.

[olivierlambert]

Great, we'll merge the fix in master ASAP (it's a public holiday in France today)

[julien-f]

@Andrew @Danp It's merged, thank you

[Andrew]

@olivierlambert @julien-f
Merci. Bonne Fête De L'Ascension.

[olivierlambert]

Thank you!