yahoo-eng-team team mailing list archive

Thread
Date
[Bug 1713495] [NEW] Nova reports various hypervisor resources

To: yahoo-eng-team@xxxxxxxxxxxxxxxxxxx
From: Jacolex <jacolex@xxxxxxxxx>
Date: Mon, 28 Aug 2017 13:12:51 -0000
Reply-to: Bug 1713495 <1713495@xxxxxxxxxxxxxxxxxx>
Sender: bounces@xxxxxxxxxxxxx
Public bug reported:

Hello
I experienced that Openstack provides two different hypervisor usage statistics (Ocata).
I checked that some nova processes reports different cpu and ram usage to database.
These are queries from mysql.log which are providing different statistics:

2017-08-28T11:29:19.037802Z       134 Query     UPDATE compute_nodes SET
updated_at='2017-08-28 11:29:19.036118', vcpus_used=126,
memory_mb_used=134660, local_gb_used=215601,
disk_available_least=275518, stats='{}',
numa_topology='{\"nova_object.version\": \"1.2\",
\"nova_object.changes\": [\"cells\"], \"nova_object.name\":
\"NUMATopology\", \"nova_object.data\": {\"cells\":
[{\"nova_object.version\": \"1.2\", \"nova_object.changes\":
[\"cpu_usage\", \"memory_usage\", \"cpuset\", \"mempages\",
\"pinned_cpus\", \"memory\", \"siblings\", \"id\"],
\"nova_object.name\": \"NUMACell\", \"nova_object.data\":
{\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [0, 1, 2, 3, 4, 5,
6, 7, 16, 17, 18, 19, 20, 21, 22, 23], \"pinned_cpus\": [],
\"siblings\": [[2, 18], [1, 17], [0, 16], [19, 3], [22, 6], [20, 4],
[23, 7], [5, 21]], \"memory\": 128908, \"mempages\":
[{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\",
\"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\":
33000563, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\":
\"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
[\"total\", \"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0,
\"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\":
\"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
[\"total\", \"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0,
\"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\":
\"nova\"}], \"id\": 0}, \"nova_object.namespace\": \"nova\"},
{\"nova_object.version\": \"1.2\", \"nova_object.changes\":
[\"cpu_usage\", \"memory_usage\", \"cpuset\", \"mempages\",
\"pinned_cpus\", \"memory\", \"siblings\", \"id\"],
\"nova_object.name\": \"NUMACell\", \"nova_object.data\":
{\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [8, 9, 10, 11, 12,
13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31], \"pinned_cpus\": [],
\"siblings\": [[8, 24], [30, 14], [31, 15], [27, 11], [10, 26], [28,
12], [9, 25], [13, 29]], \"memory\": 129021, \"mempages\":
[{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\",
\"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\":
33029464, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\":
\"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
[\"total\", \"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0,
\"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\":
\"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
[\"total\", \"used\", \"reserved\", \"size_kb\"], \"nova_object.name\":
\"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0,
\"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\":
\"nova\"}], \"id\": 1}, \"nova_object.namespace\": \"nova\"}]},
\"nova_object.namespace\": \"nova\"}' WHERE compute_nodes.id = 63


2017-08-28T11:29:20.547435Z        89 Query     UPDATE compute_nodes SET updated_at='2017-08-28 11:29:20.544879', vcpus_used=132, memory_mb_used=257536, local_gb_used=2590, stats='{\"num_proj_d1de178820ea48c3a1b098df3d4a4b7c\": \"2\", \"num_instances\": \"29\", \"num_proj_ada9ef29bef5486bb7f386ecd7a7cb2f\": \"1\", \"num_proj_dfd2237d06114848b77eb5cd4d5cb02a\": \"1\", \"num_proj_e8f91dabfe664223899bfeabdee8735f\": \"1\", \"num_proj_28c4118284f545a389eb20d745e3c5d9\": \"2\", \"io_workload\": \"0\", \"num_vm_active\": \"26\", \"num_proj_c673146ecf0349bbbe06a284c9fd2fc0\": \"1\", \"num_vm_stopped\": \"2\", \"num_proj_505ccb54aaa049199a402e3d8369bcca\": \"1\", \"num_proj_3f410526915f446c904d4aeadd97021e\": \"1\", \"num_proj_e37a451d77524a00b977538d3a58f53d\": \"1\", \"num_proj_110d63d719464a9384336ac6c9730330\": \"1\", \"num_proj_eb0d7cf7e0394dd082322d0dfe1753b4\": \"1\", \"num_proj_eeaf551320d647c18762f80a565d77e4\": \"1\", \"num_proj_3cdc8848793f428a8884809daa499af6\": \"1\", \"num_task_None\": \"29\", \"num_proj_b01521bb84fa45ce980f56a9bb5565c7\": \"1\", \"num_proj_7e689fc19cc448cf9b78ba594e9eb27e\": \"1\", \"num_proj_11bfa9122bf04381a1dd0c33ef1d0181\": \"1\", \"num_vm_suspended\": \"1\", \"num_proj_c225559f26e34070be29b8d1460b3eb7\": \"1\", \"num_proj_1977d8ae1cde41bc83e2b466c49a73ad\": \"1\", \"num_proj_2e5b2b05d443449789cd00368bd25545\": \"1\", \"num_proj_2cfcd2e3bf8f45cc89b4155061dd373e\": \"1\", \"num_proj_2f4fd3ca13d6441f9d7c8bd097377f1b\": \"2\", \"num_proj_be58cda2f3ca4c07b0162c9b13a41e88\": \"1\", \"num_proj_efb79b155383443d88dcc25c6008e170\": \"2\", \"num_os_type_None\": \"29\", \"num_proj_7fd4b8ef480d4e85b545b7c28996c473\": \"1\", \"num_proj_9f79e068e7d14d219c2b513cd9c6e83b\": \"1\"}', numa_topology='{\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cells\"], \"nova_object.name\": \"NUMATopology\", \"nova_object.data\": {\"cells\": [{\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cpu_usage\", \"memory_usage\", \"cpuset\", \"pinned_cpus\", \"siblings\", \"memory\", \"mempages\", \"id\"], \"nova_object.name\": \"NUMACell\", \"nova_object.data\": {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23], \"pinned_cpus\": [], \"siblings\": [[2, 18], [1, 17], [0, 16], [3, 19], [6, 22], [4, 20], [7, 23], [5, 21]], \"memory\": 128908, \"mempages\": [{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 33000563, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\": \"nova\"}], \"id\": 0}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cpu_usage\", \"memory_usage\", \"cpuset\", \"pinned_cpus\", \"siblings\", \"memory\", \"mempages\", \"id\"], \"nova_object.name\": \"NUMACell\", \"nova_object.data\": {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31], \"pinned_cpus\": [], \"siblings\": [[8, 24], [14, 30], [15, 31], [11, 27], [10, 26], [12, 28], [9, 25], [13, 29]], \"memory\": 129021, \"mempages\": [{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 33029464, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\": \"nova\"}], \"id\": 1}, \"nova_object.namespace\": \"nova\"}]}, \"nova_object.namespace\": \"nova\"}' WHERE compute_nodes.id = 63

First one has memory_mb_used=134660, the second has  memory_mb_used=257536.
First one is actual used memory by hypervisor host:
openstack-wn10:~# free -m
              total        used        free      shared  buff/cache   available
Mem:         257929      134238      120187        1897        3503      120788
Swap:             0           0           0

The second one is memory summary of running instances.

So memory_mb_used value changes over time from one value to another.

** Affects: nova
     Importance: Undecided
         Status: New


** Tags: nova

-- 
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1713495

Title:
  Nova reports various hypervisor resources

Status in OpenStack Compute (nova):
  New

Bug description:
  Hello
  I experienced that Openstack provides two different hypervisor usage statistics (Ocata).
  I checked that some nova processes reports different cpu and ram usage to database.
  These are queries from mysql.log which are providing different statistics:

  2017-08-28T11:29:19.037802Z       134 Query     UPDATE compute_nodes
  SET updated_at='2017-08-28 11:29:19.036118', vcpus_used=126,
  memory_mb_used=134660, local_gb_used=215601,
  disk_available_least=275518, stats='{}',
  numa_topology='{\"nova_object.version\": \"1.2\",
  \"nova_object.changes\": [\"cells\"], \"nova_object.name\":
  \"NUMATopology\", \"nova_object.data\": {\"cells\":
  [{\"nova_object.version\": \"1.2\", \"nova_object.changes\":
  [\"cpu_usage\", \"memory_usage\", \"cpuset\", \"mempages\",
  \"pinned_cpus\", \"memory\", \"siblings\", \"id\"],
  \"nova_object.name\": \"NUMACell\", \"nova_object.data\":
  {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [0, 1, 2, 3, 4, 5,
  6, 7, 16, 17, 18, 19, 20, 21, 22, 23], \"pinned_cpus\": [],
  \"siblings\": [[2, 18], [1, 17], [0, 16], [19, 3], [22, 6], [20, 4],
  [23, 7], [5, 21]], \"memory\": 128908, \"mempages\":
  [{\"nova_object.version\": \"1.1\", \"nova_object.changes\":
  [\"total\", \"used\", \"reserved\", \"size_kb\"],
  \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\":
  {\"used\": 0, \"total\": 33000563, \"reserved\": 0, \"size_kb\": 4},
  \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\":
  \"1.1\", \"nova_object.changes\": [\"total\", \"used\", \"reserved\",
  \"size_kb\"], \"nova_object.name\": \"NUMAPagesTopology\",
  \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0,
  \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"},
  {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
  [\"total\", \"used\", \"reserved\", \"size_kb\"],
  \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\":
  {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576},
  \"nova_object.namespace\": \"nova\"}], \"id\": 0},
  \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\":
  \"1.2\", \"nova_object.changes\": [\"cpu_usage\", \"memory_usage\",
  \"cpuset\", \"mempages\", \"pinned_cpus\", \"memory\", \"siblings\",
  \"id\"], \"nova_object.name\": \"NUMACell\", \"nova_object.data\":
  {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [8, 9, 10, 11, 12,
  13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31], \"pinned_cpus\": [],
  \"siblings\": [[8, 24], [30, 14], [31, 15], [27, 11], [10, 26], [28,
  12], [9, 25], [13, 29]], \"memory\": 129021, \"mempages\":
  [{\"nova_object.version\": \"1.1\", \"nova_object.changes\":
  [\"total\", \"used\", \"reserved\", \"size_kb\"],
  \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\":
  {\"used\": 0, \"total\": 33029464, \"reserved\": 0, \"size_kb\": 4},
  \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\":
  \"1.1\", \"nova_object.changes\": [\"total\", \"used\", \"reserved\",
  \"size_kb\"], \"nova_object.name\": \"NUMAPagesTopology\",
  \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0,
  \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"},
  {\"nova_object.version\": \"1.1\", \"nova_object.changes\":
  [\"total\", \"used\", \"reserved\", \"size_kb\"],
  \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\":
  {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576},
  \"nova_object.namespace\": \"nova\"}], \"id\": 1},
  \"nova_object.namespace\": \"nova\"}]}, \"nova_object.namespace\":
  \"nova\"}' WHERE compute_nodes.id = 63

  
  2017-08-28T11:29:20.547435Z        89 Query     UPDATE compute_nodes SET updated_at='2017-08-28 11:29:20.544879', vcpus_used=132, memory_mb_used=257536, local_gb_used=2590, stats='{\"num_proj_d1de178820ea48c3a1b098df3d4a4b7c\": \"2\", \"num_instances\": \"29\", \"num_proj_ada9ef29bef5486bb7f386ecd7a7cb2f\": \"1\", \"num_proj_dfd2237d06114848b77eb5cd4d5cb02a\": \"1\", \"num_proj_e8f91dabfe664223899bfeabdee8735f\": \"1\", \"num_proj_28c4118284f545a389eb20d745e3c5d9\": \"2\", \"io_workload\": \"0\", \"num_vm_active\": \"26\", \"num_proj_c673146ecf0349bbbe06a284c9fd2fc0\": \"1\", \"num_vm_stopped\": \"2\", \"num_proj_505ccb54aaa049199a402e3d8369bcca\": \"1\", \"num_proj_3f410526915f446c904d4aeadd97021e\": \"1\", \"num_proj_e37a451d77524a00b977538d3a58f53d\": \"1\", \"num_proj_110d63d719464a9384336ac6c9730330\": \"1\", \"num_proj_eb0d7cf7e0394dd082322d0dfe1753b4\": \"1\", \"num_proj_eeaf551320d647c18762f80a565d77e4\": \"1\", \"num_proj_3cdc8848793f428a8884809daa499af6\": \"1\", \"num_task_None\": \"29\", \"num_proj_b01521bb84fa45ce980f56a9bb5565c7\": \"1\", \"num_proj_7e689fc19cc448cf9b78ba594e9eb27e\": \"1\", \"num_proj_11bfa9122bf04381a1dd0c33ef1d0181\": \"1\", \"num_vm_suspended\": \"1\", \"num_proj_c225559f26e34070be29b8d1460b3eb7\": \"1\", \"num_proj_1977d8ae1cde41bc83e2b466c49a73ad\": \"1\", \"num_proj_2e5b2b05d443449789cd00368bd25545\": \"1\", \"num_proj_2cfcd2e3bf8f45cc89b4155061dd373e\": \"1\", \"num_proj_2f4fd3ca13d6441f9d7c8bd097377f1b\": \"2\", \"num_proj_be58cda2f3ca4c07b0162c9b13a41e88\": \"1\", \"num_proj_efb79b155383443d88dcc25c6008e170\": \"2\", \"num_os_type_None\": \"29\", \"num_proj_7fd4b8ef480d4e85b545b7c28996c473\": \"1\", \"num_proj_9f79e068e7d14d219c2b513cd9c6e83b\": \"1\"}', numa_topology='{\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cells\"], \"nova_object.name\": \"NUMATopology\", \"nova_object.data\": {\"cells\": [{\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cpu_usage\", \"memory_usage\", \"cpuset\", \"pinned_cpus\", \"siblings\", \"memory\", \"mempages\", \"id\"], \"nova_object.name\": \"NUMACell\", \"nova_object.data\": {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23], \"pinned_cpus\": [], \"siblings\": [[2, 18], [1, 17], [0, 16], [3, 19], [6, 22], [4, 20], [7, 23], [5, 21]], \"memory\": 128908, \"mempages\": [{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 33000563, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\": \"nova\"}], \"id\": 0}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.2\", \"nova_object.changes\": [\"cpu_usage\", \"memory_usage\", \"cpuset\", \"pinned_cpus\", \"siblings\", \"memory\", \"mempages\", \"id\"], \"nova_object.name\": \"NUMACell\", \"nova_object.data\": {\"cpu_usage\": 0, \"memory_usage\": 0, \"cpuset\": [8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31], \"pinned_cpus\": [], \"siblings\": [[8, 24], [14, 30], [15, 31], [11, 27], [10, 26], [12, 28], [9, 25], [13, 29]], \"memory\": 129021, \"mempages\": [{\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 33029464, \"reserved\": 0, \"size_kb\": 4}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 2048}, \"nova_object.namespace\": \"nova\"}, {\"nova_object.version\": \"1.1\", \"nova_object.changes\": [\"total\", \"reserved\", \"size_kb\", \"used\"], \"nova_object.name\": \"NUMAPagesTopology\", \"nova_object.data\": {\"used\": 0, \"total\": 0, \"reserved\": 0, \"size_kb\": 1048576}, \"nova_object.namespace\": \"nova\"}], \"id\": 1}, \"nova_object.namespace\": \"nova\"}]}, \"nova_object.namespace\": \"nova\"}' WHERE compute_nodes.id = 63

  First one has memory_mb_used=134660, the second has  memory_mb_used=257536.
  First one is actual used memory by hypervisor host:
  openstack-wn10:~# free -m
                total        used        free      shared  buff/cache   available
  Mem:         257929      134238      120187        1897        3503      120788
  Swap:             0           0           0

  The second one is memory summary of running instances.

  So memory_mb_used value changes over time from one value to another.

To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1713495/+subscriptions
Follow ups

[Bug 1713495] Re: Nova reports various hypervisor resources
From: Launchpad Bug Tracker, 2017-11-01