← Back to team overview

duplicity-team team mailing list archive

Re: [Question #631423]: Too many open files (again)

 

Question #631423 on Duplicity changed:
https://answers.launchpad.net/duplicity/+question/631423

    Status: Answered => Solved

Howard Kaye confirmed that the question is solved:
OK, I found and fixed the root cause of this error:

Duplicity was failing in the glue code (in C), between Python and librsync.
It did an fdopen() on the file descriptor from a Python File object, and then never
closed the File * which was returned.  Even though the Python File object (and the
file descriptor shared by the File * and the Python File) got closed when it was
deallocated, the File *'s were being leaked, and eventually the stdio library ran
out of File *'s (even though the number of open files was not that large).

The fix is to dup the file descriptor, and then close the file in the deallocator
routine in the glue code.  Duping the file lets the C code and the Python code
each close the file, when they are done with it.

*** _librsyncmodule.c.orig      2017-01-06 09:23:21.000000000 -0500
--- _librsyncmodule.c   2017-05-16 11:12:58.000000000 -0400
***************
*** 23,28 ****
--- 23,29 ----
   * ----------------------------------------------------------------------- */
  
  #include <Python.h>
+ #include <errno.h>
  #include <librsync.h>
  #define RS_JOB_BLOCKSIZE 65536
  
***************
*** 287,292 ****
--- 288,294 ----
    PyObject_HEAD
    rs_job_t *patch_job;
    PyObject *basis_file;
+   FILE *cfile;
  } _librsync_PatchMakerObject;
  
  /* Call with the basis file */
***************
*** 296,302 ****
    _librsync_PatchMakerObject* pm;
    PyObject *python_file;
    int fd;
-   FILE *cfile;
  
    if (!PyArg_ParseTuple(args, "O:new_patchmaker", &python_file))
      return NULL;
--- 298,303 ----
***************
*** 305,318 ****
      PyErr_SetString(PyExc_TypeError, "Need true file object");
      return NULL;
    }
    Py_INCREF(python_file);
  
    pm = PyObject_New(_librsync_PatchMakerObject, &_librsync_PatchMakerType);
    if (pm == NULL) return NULL;
  
    pm->basis_file = python_file;
!   cfile = fdopen(fd, "rb");
!   pm->patch_job = rs_patch_begin(rs_file_copy_cb, cfile);
  
    return (PyObject*)pm;
  }
--- 306,327 ----
      PyErr_SetString(PyExc_TypeError, "Need true file object");
      return NULL;
    }
+   /* get our own private copy of the file, so we can close it later. */
+   fd = dup(fd);
+   if (fd == -1) {
+     char buf[256];
+     strerror_r(errno, buf, sizeof(buf));
+     PyErr_SetString(PyExc_TypeError, buf);
+     return NULL;
+   }
    Py_INCREF(python_file);
  
    pm = PyObject_New(_librsync_PatchMakerObject, &_librsync_PatchMakerType);
    if (pm == NULL) return NULL;
  
    pm->basis_file = python_file;
!   pm->cfile = fdopen(fd, "rb");
!   pm->patch_job = rs_patch_begin(rs_file_copy_cb, pm->cfile);
  
    return (PyObject*)pm;
  }
***************
*** 323,328 ****
--- 332,340 ----
    _librsync_PatchMakerObject *pm = (_librsync_PatchMakerObject *)self;
    Py_DECREF(pm->basis_file);
    rs_job_free(pm->patch_job);
+   if (pm->cfile) {
+     fclose(pm->cfile);
+   }
    PyObject_Del(self);
  }

-- 
You received this question notification because your team duplicity-team
is an answer contact for Duplicity.


References