kernel-packages team mailing list archive

[Andris Zāģeris <1232473@xxxxxxxxxxxxxxxxxx>]

Public bug reported:

I ran update manager, it was installing a new kernel (3.2.0-54-generic).
Then an error dialog/message popped up that the disk (root partition)
was full, the dialog had two buttons (examine - starts baobab and
skip/cancel).

At first I tried to find any big files (du didn't help, showed that only
8GB of 20GB was used). Long story short, googled a bit about why df and
du might disagree and someone recommended lsof | grep deleted.

depmod (i think) had created a file and deleted it then kept writing to
it filling up my root partition. In the end I had to kill it, because it
just kept writing to it even when I zeroed out that deleted file.

Here's a log of what I typed in the console:

root@localhost:/tmp/fooroot# lsof | grep deleted
lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
      Output information may be incomplete.
rsyslogd   1046     syslog  txt       REG                8,1      385048     541758 /usr/sbin/rsyslogd (deleted)
compiz     2133     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
compiz     2133     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
nautilus   2149     andris   14r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
nautilus   2149     andris   15r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
gvfsd-tra  2222     andris    9r      REG               0,21      132484    2236225 /home/andris/.local/share/gvfs-metadata/home (deleted)
gvfsd-tra  2222     andris   17r      REG               0,21       32768    2236270 /home/andris/.local/share/gvfs-metadata/home-e8e5eff2.log (deleted)
gvfsd-tra  2222     andris   18r      REG               0,21       63236    2236055 /home/andris/.local/share/gvfs-metadata/root (deleted)
gvfsd-tra  2222     andris   19r      REG               0,21       32768    2236104 /home/andris/.local/share/gvfs-metadata/root-da97291d.log (deleted)
bamfdaemo  2225     andris   12r      REG               0,21       62912    2229546 /home/andris/.local/share/gvfs-metadata/root (deleted)
bamfdaemo  2225     andris   13r      REG               0,21       32768    2236924 /home/andris/.local/share/gvfs-metadata/root-691fc386.log (deleted)
sh         2246     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
sh         2246     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
gtk-windo  2247     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
gtk-windo  2247     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
update-ma  2585     andris   18u      REG               0,17         524      27997 /tmp/vteKY8D4W (deleted)
update-ma  2585     andris   19u      REG               0,17         176      27998 /tmp/vtePX8D4W (deleted)
update-ma  2585     andris  108r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
aptd       3245       root   10r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
aptd       3269       root   10r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/tmp/fooroot# lsof | grep deleted | grep depmod
lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
      Output information may be incomplete.
depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/tmp/fooroot# sudo lsof | grep deleted | grep depmod
lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
      Output information may be incomplete.
depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/tmp/fooroot# cd cd /proc/8673/fd
bash: cd: cd: No such file or directory
root@localhost:/tmp/fooroot# cd /proc/8673/fd
root@localhost:/proc/8673/fd# ls
0  1  102  2  3
root@localhost:/proc/8673/fd# ll -h
total 0
dr-x------ 2 root root  0 sep 28 16:52 ./
dr-xr-xr-x 8 root root  0 sep 28 16:01 ../
lr-x------ 1 root root 64 sep 28 16:52 0 -> /dev/null
l-wx------ 1 root root 64 sep 28 16:52 1 -> /dev/null
lrwx------ 1 root root 64 sep 28 16:52 102 -> /dev/pts/2
l-wx------ 1 root root 64 sep 28 16:52 2 -> /dev/null
l-wx------ 1 root root 64 sep 28 16:52 3 -> /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/proc/8673/fd# > 3
root@localhost:/proc/8673/fd# df
Filesystem            1K-blocks     Used Available Use% Mounted on
/dev/sda1              23067108  9398176  12497172  43% /
udev                    1958476       12   1958464   1% /dev
none                    1970340     3960   1966380   1% /tmp
tmpfs                    788140      916    787224   1% /run
none                       5120        0      5120   0% /run/lock
none                    1970340      196   1970144   1% /run/shm
/dev/sda2              96121612 40710888  50527936  45% /home
/home/andris/.Private  96121612 40710888  50527936  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G  9,3G   12G  45% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# sudo lsof | grep deleted | grep depmod
lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
      Output information may be incomplete.
depmod     8673       root    3w      REG                8,1 27424661504     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/proc/8673/fd# ll -h
total 0
dr-x------ 2 root root  0 sep 28 16:52 ./
dr-xr-xr-x 8 root root  0 sep 28 16:01 ../
lr-x------ 1 root root 64 sep 28 17:33 0 -> /dev/null
l-wx------ 1 root root 64 sep 28 17:33 1 -> /dev/null
lrwx------ 1 root root 64 sep 28 17:33 102 -> /dev/pts/2
l-wx------ 1 root root 64 sep 28 17:33 2 -> /dev/null
l-wx------ 1 root root 64 sep 28 17:33 3 -> /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G   22G     0 100% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# > 3
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G  8,8G   13G  42% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G  8,9G   13G  43% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G  8,9G   13G  43% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G  9,7G   12G  47% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G   10G   11G  48% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# df -h
Filesystem             Size  Used Avail Use% Mounted on
/dev/sda1               22G   11G   11G  49% /
udev                   1,9G   12K  1,9G   1% /dev
none                   1,9G  3,9M  1,9G   1% /tmp
tmpfs                  770M  916K  769M   1% /run
none                   5,0M     0  5,0M   0% /run/lock
none                   1,9G  196K  1,9G   1% /run/shm
/dev/sda2               92G   39G   49G  45% /home
/home/andris/.Private   92G   39G   49G  45% /home/andris
root@localhost:/proc/8673/fd# > 3
root@localhost:/proc/8673/fd# pgrep depmod
8673
root@localhost:/proc/8673/fd# pkill depmod
root@localhost:/proc/8673/fd# pgrep depmod
28300


Seems I'm not the only one who's seen this issue with kernel updates: http://askubuntu.com/questions/157740/updating-to-linux-image-3-2-0-26-generic-occupies-all-disk-space

** Affects: linux (Ubuntu)
     Importance: Undecided
         Status: Incomplete


** Tags: precise

-- 
You received this bug notification because you are a member of Kernel
Packages, which is subscribed to linux in Ubuntu.
https://bugs.launchpad.net/bugs/1232473

Title:
  depmod fills disk to 100% with invisible modules.ccwmap

Status in “linux” package in Ubuntu:
  Incomplete

Bug description:
  I ran update manager, it was installing a new kernel
  (3.2.0-54-generic). Then an error dialog/message popped up that the
  disk (root partition) was full, the dialog had two buttons (examine -
  starts baobab and skip/cancel).

  At first I tried to find any big files (du didn't help, showed that
  only 8GB of 20GB was used). Long story short, googled a bit about why
  df and du might disagree and someone recommended lsof | grep deleted.

  depmod (i think) had created a file and deleted it then kept writing
  to it filling up my root partition. In the end I had to kill it,
  because it just kept writing to it even when I zeroed out that deleted
  file.

  Here's a log of what I typed in the console:

  root@localhost:/tmp/fooroot# lsof | grep deleted
  lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
        Output information may be incomplete.
  rsyslogd   1046     syslog  txt       REG                8,1      385048     541758 /usr/sbin/rsyslogd (deleted)
  compiz     2133     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
  compiz     2133     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
  nautilus   2149     andris   14r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
  nautilus   2149     andris   15r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
  gvfsd-tra  2222     andris    9r      REG               0,21      132484    2236225 /home/andris/.local/share/gvfs-metadata/home (deleted)
  gvfsd-tra  2222     andris   17r      REG               0,21       32768    2236270 /home/andris/.local/share/gvfs-metadata/home-e8e5eff2.log (deleted)
  gvfsd-tra  2222     andris   18r      REG               0,21       63236    2236055 /home/andris/.local/share/gvfs-metadata/root (deleted)
  gvfsd-tra  2222     andris   19r      REG               0,21       32768    2236104 /home/andris/.local/share/gvfs-metadata/root-da97291d.log (deleted)
  bamfdaemo  2225     andris   12r      REG               0,21       62912    2229546 /home/andris/.local/share/gvfs-metadata/root (deleted)
  bamfdaemo  2225     andris   13r      REG               0,21       32768    2236924 /home/andris/.local/share/gvfs-metadata/root-691fc386.log (deleted)
  sh         2246     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
  sh         2246     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
  gtk-windo  2247     andris    9r      REG               0,21      133516    2235018 /home/andris/.local/share/gvfs-metadata/home (deleted)
  gtk-windo  2247     andris   10r      REG               0,21       32768    2236823 /home/andris/.local/share/gvfs-metadata/home-a5eca004.log (deleted)
  update-ma  2585     andris   18u      REG               0,17         524      27997 /tmp/vteKY8D4W (deleted)
  update-ma  2585     andris   19u      REG               0,17         176      27998 /tmp/vtePX8D4W (deleted)
  update-ma  2585     andris  108r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
  aptd       3245       root   10r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
  aptd       3269       root   10r      REG                8,1     2300543     134094 /var/lib/dpkg/status (deleted)
  depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/tmp/fooroot# lsof | grep deleted | grep depmod
  lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
        Output information may be incomplete.
  depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/tmp/fooroot# sudo lsof | grep deleted | grep depmod
  lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
        Output information may be incomplete.
  depmod     8673       root    3w      REG                8,1 14525296640     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/tmp/fooroot# cd cd /proc/8673/fd
  bash: cd: cd: No such file or directory
  root@localhost:/tmp/fooroot# cd /proc/8673/fd
  root@localhost:/proc/8673/fd# ls
  0  1  102  2  3
  root@localhost:/proc/8673/fd# ll -h
  total 0
  dr-x------ 2 root root  0 sep 28 16:52 ./
  dr-xr-xr-x 8 root root  0 sep 28 16:01 ../
  lr-x------ 1 root root 64 sep 28 16:52 0 -> /dev/null
  l-wx------ 1 root root 64 sep 28 16:52 1 -> /dev/null
  lrwx------ 1 root root 64 sep 28 16:52 102 -> /dev/pts/2
  l-wx------ 1 root root 64 sep 28 16:52 2 -> /dev/null
  l-wx------ 1 root root 64 sep 28 16:52 3 -> /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/proc/8673/fd# > 3
  root@localhost:/proc/8673/fd# df
  Filesystem            1K-blocks     Used Available Use% Mounted on
  /dev/sda1              23067108  9398176  12497172  43% /
  udev                    1958476       12   1958464   1% /dev
  none                    1970340     3960   1966380   1% /tmp
  tmpfs                    788140      916    787224   1% /run
  none                       5120        0      5120   0% /run/lock
  none                    1970340      196   1970144   1% /run/shm
  /dev/sda2              96121612 40710888  50527936  45% /home
  /home/andris/.Private  96121612 40710888  50527936  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G  9,3G   12G  45% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# sudo lsof | grep deleted | grep depmod
  lsof: WARNING: can't stat() fuse.gvfs-fuse-daemon file system /home/andris/.gvfs
        Output information may be incomplete.
  depmod     8673       root    3w      REG                8,1 27424661504     676388 /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/proc/8673/fd# ll -h
  total 0
  dr-x------ 2 root root  0 sep 28 16:52 ./
  dr-xr-xr-x 8 root root  0 sep 28 16:01 ../
  lr-x------ 1 root root 64 sep 28 17:33 0 -> /dev/null
  l-wx------ 1 root root 64 sep 28 17:33 1 -> /dev/null
  lrwx------ 1 root root 64 sep 28 17:33 102 -> /dev/pts/2
  l-wx------ 1 root root 64 sep 28 17:33 2 -> /dev/null
  l-wx------ 1 root root 64 sep 28 17:33 3 -> /lib/modules/3.2.0-54-generic/modules.ccwmap (deleted)
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G   22G     0 100% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# > 3
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G  8,8G   13G  42% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G  8,9G   13G  43% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G  8,9G   13G  43% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G  9,7G   12G  47% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G   10G   11G  48% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# df -h
  Filesystem             Size  Used Avail Use% Mounted on
  /dev/sda1               22G   11G   11G  49% /
  udev                   1,9G   12K  1,9G   1% /dev
  none                   1,9G  3,9M  1,9G   1% /tmp
  tmpfs                  770M  916K  769M   1% /run
  none                   5,0M     0  5,0M   0% /run/lock
  none                   1,9G  196K  1,9G   1% /run/shm
  /dev/sda2               92G   39G   49G  45% /home
  /home/andris/.Private   92G   39G   49G  45% /home/andris
  root@localhost:/proc/8673/fd# > 3
  root@localhost:/proc/8673/fd# pgrep depmod
  8673
  root@localhost:/proc/8673/fd# pkill depmod
  root@localhost:/proc/8673/fd# pgrep depmod
  28300

  
  Seems I'm not the only one who's seen this issue with kernel updates: http://askubuntu.com/questions/157740/updating-to-linux-image-3-2-0-26-generic-occupies-all-disk-space

To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1232473/+subscriptions