[1/2] lock: Import file lock class from mercurial

Message ID 1445342923-6310-2-git-send-email-damien.lespiau@intel.com
State Rejected
Headers show

Commit Message

Damien Lespiau Oct. 20, 2015, 12:08 p.m.
With series now in production, I realized I needed the mail parsing to
be atomic. Because of the race can happen between mutiple process a file
lock seems like it could work.

Signed-off-by: Damien Lespiau <damien.lespiau@intel.com>
---
 patchwork/lock.py            | 301 +++++++++++++++++++++++++++++++++++++++++++
 patchwork/tests/test_lock.py | 290 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 591 insertions(+)
 create mode 100644 patchwork/lock.py
 create mode 100644 patchwork/tests/test_lock.py

Patch

diff --git a/patchwork/lock.py b/patchwork/lock.py
new file mode 100644
index 0000000..41ee972
--- /dev/null
+++ b/patchwork/lock.py
@@ -0,0 +1,301 @@ 
+# lock.py - simple advisory locking scheme for mercurial
+#
+# Copyright 2005, 2006 Matt Mackall <mpm@selenic.com>
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+#
+
+# This file has been taken from the mercurial project, the base revision it's
+# derived from is:
+#     https://selenic.com/hg/rev/e8564e04382d
+# A few changes have been made:
+#   - revert to not using the vfs object
+#   - import a few functions and classes from util.py and error.py
+
+from __future__ import absolute_import
+
+import contextlib
+import errno
+import os
+import socket
+import time
+import warnings
+
+#
+# from error.py
+#
+
+class LockError(IOError):
+    def __init__(self, errno, strerror, filename, desc):
+        IOError.__init__(self, errno, strerror, filename)
+        self.desc = desc
+
+class LockHeld(LockError):
+    def __init__(self, errno, filename, desc, locker):
+        LockError.__init__(self, errno, 'Lock held', filename, desc)
+        self.locker = locker
+
+class LockUnavailable(LockError):
+    pass
+
+# LockError is for errors while acquiring the lock -- this is unrelated
+class LockInheritanceContractViolation(RuntimeError):
+    pass
+
+#
+# from util.py
+#
+
+def testpid(pid):
+    '''return False if pid dead, True if running or not sure'''
+    if os.sys.platform == 'OpenVMS':
+        return True
+    try:
+        os.kill(pid, 0)
+        return True
+    except OSError as inst:
+        return inst.errno != errno.ESRCH
+
+def makelock(info, pathname):
+    try:
+        return os.symlink(info, pathname)
+    except OSError as why:
+        if why.errno == errno.EEXIST:
+            raise
+    except AttributeError: # no symlink in os
+        pass
+
+    ld = os.open(pathname, os.O_CREAT | os.O_WRONLY | os.O_EXCL)
+    os.write(ld, info)
+    os.close(ld)
+
+def readlock(pathname):
+    try:
+        return os.readlink(pathname)
+    except OSError as why:
+        if why.errno not in (errno.EINVAL, errno.ENOSYS):
+            raise
+    except AttributeError: # no symlink in os
+        pass
+    fp = posixfile(pathname)
+    r = fp.read()
+    fp.close()
+    return r
+
+#
+# from lock.py
+#
+
+class lock(object):
+    '''An advisory lock held by one process to control access to a set
+    of files.  Non-cooperating processes or incorrectly written scripts
+    can ignore Mercurial's locking scheme and stomp all over the
+    repository, so don't do that.
+
+    Typically used via localrepository.lock() to lock the repository
+    store (.hg/store/) or localrepository.wlock() to lock everything
+    else under .hg/.'''
+
+    # lock is symlink on platforms that support it, file on others.
+
+    # symlink is used because create of directory entry and contents
+    # are atomic even over nfs.
+
+    # old-style lock: symlink to pid
+    # new-style lock: symlink to hostname:pid
+
+    _host = None
+
+    def __init__(self, file, timeout=-1, releasefn=None, acquirefn=None,
+                 desc=None, inheritchecker=None, parentlock=None):
+        self.f = file
+        self.held = 0
+        self.timeout = timeout
+        self.releasefn = releasefn
+        self.acquirefn = acquirefn
+        self.desc = desc
+        self._inheritchecker = inheritchecker
+        self.parentlock = parentlock
+        self._parentheld = False
+        self._inherited = False
+        self.postrelease  = []
+        self.pid = self._getpid()
+        self.delay = self.lock()
+        if self.acquirefn:
+            self.acquirefn()
+
+    def __del__(self):
+        if self.held:
+            warnings.warn("use lock.release instead of del lock",
+                    category=DeprecationWarning,
+                    stacklevel=2)
+
+            # ensure the lock will be removed
+            # even if recursive locking did occur
+            self.held = 1
+
+        self.release()
+
+    def _getpid(self):
+        # wrapper around os.getpid() to make testing easier
+        return os.getpid()
+
+    def lock(self):
+        timeout = self.timeout
+        while True:
+            try:
+                self._trylock()
+                return self.timeout - timeout
+            except LockHeld as inst:
+                if timeout != 0:
+                    time.sleep(1)
+                    if timeout > 0:
+                        timeout -= 1
+                    continue
+                raise LockHeld(errno.ETIMEDOUT, inst.filename, self.desc,
+                               inst.locker)
+
+    def _trylock(self):
+        if self.held:
+            self.held += 1
+            return
+        if lock._host is None:
+            lock._host = socket.gethostname()
+        lockname = '%s:%s' % (lock._host, self.pid)
+        retry = 5
+        while not self.held and retry:
+            retry -= 1
+            try:
+                makelock(lockname, self.f)
+                self.held = 1
+            except (OSError, IOError) as why:
+                if why.errno == errno.EEXIST:
+                    locker = self._readlock()
+                    # special case where a parent process holds the lock -- this
+                    # is different from the pid being different because we do
+                    # want the unlock and postrelease functions to be called,
+                    # but the lockfile to not be removed.
+                    if locker == self.parentlock:
+                        self._parentheld = True
+                        self.held = 1
+                        return
+                    locker = self._testlock(locker)
+                    if locker is not None:
+                        raise LockHeld(errno.EAGAIN, self.f, self.desc, locker)
+                else:
+                    raise LockUnavailable(why.errno, why.strerror,
+                                          why.filename, self.desc)
+
+    def _readlock(self):
+        """read lock and return its value
+
+        Returns None if no lock exists, pid for old-style locks, and host:pid
+        for new-style locks.
+        """
+        try:
+            return readlock(self.f)
+        except (OSError, IOError) as why:
+            if why.errno == errno.ENOENT:
+                return None
+            raise
+
+    def _testlock(self, locker):
+        if locker is None:
+            return None
+        try:
+            host, pid = locker.split(":", 1)
+        except ValueError:
+            return locker
+        if host != lock._host:
+            return locker
+        try:
+            pid = int(pid)
+        except ValueError:
+            return locker
+        if testpid(pid):
+            return locker
+        # if locker dead, break lock.  must do this with another lock
+        # held, or can race and break valid lock.
+        try:
+            l = lock(self.f + '.break', timeout=0)
+            os.unlink(self.f)
+            l.release()
+        except LockError:
+            return locker
+
+    def testlock(self):
+        """return id of locker if lock is valid, else None.
+
+        If old-style lock, we cannot tell what machine locker is on.
+        with new-style lock, if locker is on this machine, we can
+        see if locker is alive.  If locker is on this machine but
+        not alive, we can safely break lock.
+
+        The lock file is only deleted when None is returned.
+
+        """
+        locker = self._readlock()
+        return self._testlock(locker)
+
+    @contextlib.contextmanager
+    def inherit(self):
+        """context for the lock to be inherited by a Mercurial subprocess.
+
+        Yields a string that will be recognized by the lock in the subprocess.
+        Communicating this string to the subprocess needs to be done separately
+        -- typically by an environment variable.
+        """
+        if not self.held:
+            raise LockInheritanceContractViolation(
+                'inherit can only be called while lock is held')
+        if self._inherited:
+            raise error.LockInheritanceContractViolation(
+                'inherit cannot be called while lock is already inherited')
+        if self._inheritchecker is not None:
+            self._inheritchecker()
+        if self.releasefn:
+            self.releasefn()
+        if self._parentheld:
+            lockname = self.parentlock
+        else:
+            lockname = '%s:%s' % (lock._host, self.pid)
+        self._inherited = True
+        try:
+            yield lockname
+        finally:
+            if self.acquirefn:
+                self.acquirefn()
+            self._inherited = False
+
+    def release(self):
+        """release the lock and execute callback function if any
+
+        If the lock has been acquired multiple times, the actual release is
+        delayed to the last release call."""
+        if self.held > 1:
+            self.held -= 1
+        elif self.held == 1:
+            self.held = 0
+            if self._getpid() != self.pid:
+                # we forked, and are not the parent
+                return
+            try:
+                if self.releasefn:
+                    self.releasefn()
+            finally:
+                if not self._parentheld:
+                    try:
+                        os.unlink(self.f)
+                    except OSError:
+                        pass
+            # The postrelease functions typically assume the lock is not held
+            # at all.
+            if not self._parentheld:
+                for callback in self.postrelease:
+                    callback()
+
+def release(*locks):
+    for lock in locks:
+        if lock is not None:
+            lock.release()
diff --git a/patchwork/tests/test_lock.py b/patchwork/tests/test_lock.py
new file mode 100644
index 0000000..cd05f7f
--- /dev/null
+++ b/patchwork/tests/test_lock.py
@@ -0,0 +1,290 @@ 
+# Patchwork - automated patch tracking system
+# Copyright (C) 2015 Intel Corporation
+#
+# This file is part of the Patchwork package.
+#
+# Patchwork is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# Patchwork is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchwork; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+# This file is import from mercurial (tests/test-lock.py) with a few
+# modifications to work with patchwork's lock.py version.
+#
+# The last revision it was synced with is:
+# https://selenic.com/hg/rev/e72b62b154b0
+
+from __future__ import absolute_import
+
+import copy
+import os
+import tempfile
+import types
+import unittest
+
+from patchwork import lock
+from patchwork import lock as error
+
+testlockname = 'testlock'
+
+# work around http://bugs.python.org/issue1515
+if types.MethodType not in copy._deepcopy_dispatch:
+    def _deepcopy_method(x, memo):
+        return type(x)(x.im_func, copy.deepcopy(x.im_self, memo), x.im_class)
+    copy._deepcopy_dispatch[types.MethodType] = _deepcopy_method
+
+class lockwrapper(lock.lock):
+    def __init__(self, pidoffset, *args, **kwargs):
+        # lock.lock.__init__() calls lock(), so the pidoffset assignment needs
+        # to be earlier
+        self._pidoffset = pidoffset
+        super(lockwrapper, self).__init__(*args, **kwargs)
+    def _getpid(self):
+        return os.getpid() + self._pidoffset
+
+class teststate(object):
+    def __init__(self, testcase, dir, pidoffset=0):
+        self._testcase = testcase
+        self._acquirecalled = False
+        self._releasecalled = False
+        self._postreleasecalled = False
+        self._pidoffset = pidoffset
+
+    def makelock(self, *args, **kwargs):
+        l = lockwrapper(self._pidoffset, testlockname, releasefn=self.releasefn,
+                        acquirefn=self.acquirefn, *args, **kwargs)
+        l.postrelease.append(self.postreleasefn)
+        return l
+
+    def acquirefn(self):
+        self._acquirecalled = True
+
+    def releasefn(self):
+        self._releasecalled = True
+
+    def postreleasefn(self):
+        self._postreleasecalled = True
+
+    def assertacquirecalled(self, called):
+        self._testcase.assertEqual(
+            self._acquirecalled, called,
+            'expected acquire to be %s but was actually %s' % (
+                self._tocalled(called),
+                self._tocalled(self._acquirecalled),
+            ))
+
+    def resetacquirefn(self):
+        self._acquirecalled = False
+
+    def assertreleasecalled(self, called):
+        self._testcase.assertEqual(
+            self._releasecalled, called,
+            'expected release to be %s but was actually %s' % (
+                self._tocalled(called),
+                self._tocalled(self._releasecalled),
+            ))
+
+    def assertpostreleasecalled(self, called):
+        self._testcase.assertEqual(
+            self._postreleasecalled, called,
+            'expected postrelease to be %s but was actually %s' % (
+                self._tocalled(called),
+                self._tocalled(self._postreleasecalled),
+            ))
+
+    def assertlockexists(self, exists):
+        actual = os.path.lexists(testlockname)
+        self._testcase.assertEqual(
+            actual, exists,
+            'expected lock to %s but actually did %s' % (
+                self._toexists(exists),
+                self._toexists(actual),
+            ))
+
+    def _tocalled(self, called):
+        if called:
+            return 'called'
+        else:
+            return 'not called'
+
+    def _toexists(self, exists):
+        if exists:
+            return 'exist'
+        else:
+            return 'not exist'
+
+class testlock(unittest.TestCase):
+    def testlock(self):
+        state = teststate(self, tempfile.mkdtemp(dir=os.getcwd()))
+        lock = state.makelock()
+        state.assertacquirecalled(True)
+        lock.release()
+        state.assertreleasecalled(True)
+        state.assertpostreleasecalled(True)
+        state.assertlockexists(False)
+
+    def testrecursivelock(self):
+        state = teststate(self, tempfile.mkdtemp(dir=os.getcwd()))
+        lock = state.makelock()
+        state.assertacquirecalled(True)
+
+        state.resetacquirefn()
+        lock.lock()
+        # recursive lock should not call acquirefn again
+        state.assertacquirecalled(False)
+
+        lock.release() # brings lock refcount down from 2 to 1
+        state.assertreleasecalled(False)
+        state.assertpostreleasecalled(False)
+        state.assertlockexists(True)
+
+        lock.release() # releases the lock
+        state.assertreleasecalled(True)
+        state.assertpostreleasecalled(True)
+        state.assertlockexists(False)
+
+    def testlockfork(self):
+        state = teststate(self, tempfile.mkdtemp(dir=os.getcwd()))
+        lock = state.makelock()
+        state.assertacquirecalled(True)
+
+        # fake a fork
+        forklock = copy.deepcopy(lock)
+        forklock._pidoffset = 1
+        forklock.release()
+        state.assertreleasecalled(False)
+        state.assertpostreleasecalled(False)
+        state.assertlockexists(True)
+
+        # release the actual lock
+        lock.release()
+        state.assertreleasecalled(True)
+        state.assertpostreleasecalled(True)
+        state.assertlockexists(False)
+
+    def testinheritlock(self):
+        d = tempfile.mkdtemp(dir=os.getcwd())
+        parentstate = teststate(self, d)
+        parentlock = parentstate.makelock()
+        parentstate.assertacquirecalled(True)
+
+        # set up lock inheritance
+        with parentlock.inherit() as lockname:
+            parentstate.assertreleasecalled(True)
+            parentstate.assertpostreleasecalled(False)
+            parentstate.assertlockexists(True)
+
+            childstate = teststate(self, d, pidoffset=1)
+            childlock = childstate.makelock(parentlock=lockname)
+            childstate.assertacquirecalled(True)
+
+            childlock.release()
+            childstate.assertreleasecalled(True)
+            childstate.assertpostreleasecalled(False)
+            childstate.assertlockexists(True)
+
+            parentstate.resetacquirefn()
+
+        parentstate.assertacquirecalled(True)
+
+        parentlock.release()
+        parentstate.assertreleasecalled(True)
+        parentstate.assertpostreleasecalled(True)
+        parentstate.assertlockexists(False)
+
+    def testmultilock(self):
+        d = tempfile.mkdtemp(dir=os.getcwd())
+        state0 = teststate(self, d)
+        lock0 = state0.makelock()
+        state0.assertacquirecalled(True)
+
+        with lock0.inherit() as lock0name:
+            state0.assertreleasecalled(True)
+            state0.assertpostreleasecalled(False)
+            state0.assertlockexists(True)
+
+            state1 = teststate(self, d, pidoffset=1)
+            lock1 = state1.makelock(parentlock=lock0name)
+            state1.assertacquirecalled(True)
+
+            # from within lock1, acquire another lock
+            with lock1.inherit() as lock1name:
+                # since the file on disk is lock0's this should have the same
+                # name
+                self.assertEqual(lock0name, lock1name)
+
+                state2 = teststate(self, d, pidoffset=2)
+                lock2 = state2.makelock(parentlock=lock1name)
+                state2.assertacquirecalled(True)
+
+                lock2.release()
+                state2.assertreleasecalled(True)
+                state2.assertpostreleasecalled(False)
+                state2.assertlockexists(True)
+
+                state1.resetacquirefn()
+
+            state1.assertacquirecalled(True)
+
+            lock1.release()
+            state1.assertreleasecalled(True)
+            state1.assertpostreleasecalled(False)
+            state1.assertlockexists(True)
+
+        lock0.release()
+
+    def testinheritlockfork(self):
+        d = tempfile.mkdtemp(dir=os.getcwd())
+        parentstate = teststate(self, d)
+        parentlock = parentstate.makelock()
+        parentstate.assertacquirecalled(True)
+
+        # set up lock inheritance
+        with parentlock.inherit() as lockname:
+            childstate = teststate(self, d, pidoffset=1)
+            childlock = childstate.makelock(parentlock=lockname)
+            childstate.assertacquirecalled(True)
+
+            # fork the child lock
+            forkchildlock = copy.deepcopy(childlock)
+            forkchildlock._pidoffset += 1
+            forkchildlock.release()
+            childstate.assertreleasecalled(False)
+            childstate.assertpostreleasecalled(False)
+            childstate.assertlockexists(True)
+
+            # release the child lock
+            childlock.release()
+            childstate.assertreleasecalled(True)
+            childstate.assertpostreleasecalled(False)
+            childstate.assertlockexists(True)
+
+        parentlock.release()
+
+    def testinheritcheck(self):
+        d = tempfile.mkdtemp(dir=os.getcwd())
+        state = teststate(self, d)
+        def check():
+            raise error.LockInheritanceContractViolation('check failed')
+        lock = state.makelock(inheritchecker=check)
+        state.assertacquirecalled(True)
+
+        def tryinherit():
+            with lock.inherit():
+                pass
+
+        self.assertRaises(error.LockInheritanceContractViolation, tryinherit)
+
+        lock.release()
+
+if __name__ == '__main__':
+    unittest.main(__name__)