diff mbox

[RFC,v7,12/21] replay: recording and replaying clock ticks

Message ID 20150112120054.3504.35220.stgit@PASHA-ISP
State New
Headers show

Commit Message

Pavel Dovgalyuk Jan. 12, 2015, noon UTC
Clock ticks are considered as the sources of non-deterministic data for
virtual machine. This patch implements saving the clock values when they
are acquired (virtual, host clock, rdtsc, and some other timers).
When replaying the execution corresponding values are read from log and
transfered to the module, which wants to read the values.
Such a design required the clock polling to be synchronized. Sometimes
it is not true - e.g. when timeouts for timer lists are checked. In this case
we use a cached value of the clock, passing it to the client code.

Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@ispras.ru>
---
 cpus.c                   |    3 +-
 include/qemu/timer.h     |   10 ++++++
 qemu-timer.c             |    7 ++--
 replay/Makefile.objs     |    1 +
 replay/replay-internal.h |   11 ++++++
 replay/replay-time.c     |   79 ++++++++++++++++++++++++++++++++++++++++++++++
 replay/replay.h          |   23 +++++++++++++
 stubs/replay.c           |    9 +++++
 8 files changed, 139 insertions(+), 4 deletions(-)
 create mode 100755 replay/replay-time.c

Comments

Paolo Bonzini Jan. 12, 2015, 12:14 p.m. UTC | #1
On 12/01/2015 13:00, Pavel Dovgalyuk wrote:
> +/*! Reads next clock event from the input. */
> +int64_t replay_read_clock(unsigned int kind)
> +{
> +    if (kind >= REPLAY_CLOCK_COUNT) {
> +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
> +        exit(1);
> +    }
> +
> +    replay_exec_instructions();
> +
> +    if (replay_file) {
> +        if (skip_async_events(EVENT_CLOCK + kind)) {
> +            replay_read_next_clock(kind);
> +        }
> +        int64_t ret = replay_state.cached_clock[kind];
> +
> +        return ret;
> +    }
> +
> +    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
> +    exit(1);
> +}

Is this thread safe?  Please introduce the record/replay QemuMutex in
patch 4, and all along the series document which variables it protects.

Paolo

> diff --git a/replay/replay.h b/replay/replay.h
> index 90a949b..1c18c0e 100755
> --- a/replay/replay.h
> +++ b/replay/replay.h
> @@ -16,6 +16,16 @@
>  #include <stdint.h>
>  #include "qapi-types.h"
>  
> +/* replay clock kinds */
> +/* rdtsc */
> +#define REPLAY_CLOCK_REAL_TICKS 0
> +/* host_clock */
> +#define REPLAY_CLOCK_HOST       1
> +/* virtual_rt_clock */
> +#define REPLAY_CLOCK_VIRTUAL_RT 2
> +
> +#define REPLAY_CLOCK_COUNT      3
> +
>  extern ReplayMode replay_mode;
>  extern char *replay_image_suffix;
>  
> @@ -47,6 +57,19 @@ bool replay_interrupt(void);
>      Returns true, when interrupt request is pending */
>  bool replay_has_interrupt(void);
>  
> +/* Processing clocks and other time sources */
> +
> +/*! Save the specified clock */
> +int64_t replay_save_clock(unsigned int kind, int64_t clock);
> +/*! Read the specified clock from the log or return cached data */
> +int64_t replay_read_clock(unsigned int kind);
> +/*! Saves or reads the clock depending on the current replay mode. */
> +#define REPLAY_CLOCK(clock, value)                                      \
> +    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
> +        : replay_mode == REPLAY_MODE_RECORD                             \
> +            ? replay_save_clock((clock), (value))                       \
> +        : (value))
> +

Inline functions please, not macros.
Pavel Dovgalyuk Jan. 12, 2015, 12:43 p.m. UTC | #2
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 12/01/2015 13:00, Pavel Dovgalyuk wrote:
> > diff --git a/replay/replay.h b/replay/replay.h
> > index 90a949b..1c18c0e 100755
> > --- a/replay/replay.h
> > +++ b/replay/replay.h
> > @@ -16,6 +16,16 @@
> >  #include <stdint.h>
> >  #include "qapi-types.h"
> >
> > +/* replay clock kinds */
> > +/* rdtsc */
> > +#define REPLAY_CLOCK_REAL_TICKS 0
> > +/* host_clock */
> > +#define REPLAY_CLOCK_HOST       1
> > +/* virtual_rt_clock */
> > +#define REPLAY_CLOCK_VIRTUAL_RT 2
> > +
> > +#define REPLAY_CLOCK_COUNT      3
> > +
> >  extern ReplayMode replay_mode;
> >  extern char *replay_image_suffix;
> >
> > @@ -47,6 +57,19 @@ bool replay_interrupt(void);
> >      Returns true, when interrupt request is pending */
> >  bool replay_has_interrupt(void);
> >
> > +/* Processing clocks and other time sources */
> > +
> > +/*! Save the specified clock */
> > +int64_t replay_save_clock(unsigned int kind, int64_t clock);
> > +/*! Read the specified clock from the log or return cached data */
> > +int64_t replay_read_clock(unsigned int kind);
> > +/*! Saves or reads the clock depending on the current replay mode. */
> > +#define REPLAY_CLOCK(clock, value)                                      \
> > +    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
> > +        : replay_mode == REPLAY_MODE_RECORD                             \
> > +            ? replay_save_clock((clock), (value))                       \
> > +        : (value))
> > +
> 
> Inline functions please, not macros.

Macro is required here, because I do not want the "value" to be computed in
replay mode at all.

Pavel Dovgalyuk
Paolo Bonzini Jan. 12, 2015, 12:45 p.m. UTC | #3
On 12/01/2015 13:43, Pavel Dovgaluk wrote:
>>> > > +#define REPLAY_CLOCK(clock, value)                                      \
>>> > > +    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
>>> > > +        : replay_mode == REPLAY_MODE_RECORD                             \
>>> > > +            ? replay_save_clock((clock), (value))                       \
>>> > > +        : (value))
>>> > > +
>> > 
>> > Inline functions please, not macros.
> Macro is required here, because I do not want the "value" to be computed in
> replay mode at all.

Indeed you're right.

Paolo
Pavel Dovgalyuk Jan. 13, 2015, 9:21 a.m. UTC | #4
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 12/01/2015 13:00, Pavel Dovgalyuk wrote:
> > +/*! Reads next clock event from the input. */
> > +int64_t replay_read_clock(unsigned int kind)
> > +{
> > +    if (kind >= REPLAY_CLOCK_COUNT) {
> > +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
> > +        exit(1);
> > +    }
> > +
> > +    replay_exec_instructions();
> > +
> > +    if (replay_file) {
> > +        if (skip_async_events(EVENT_CLOCK + kind)) {
> > +            replay_read_next_clock(kind);
> > +        }
> > +        int64_t ret = replay_state.cached_clock[kind];
> > +
> > +        return ret;
> > +    }
> > +
> > +    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
> > +    exit(1);
> > +}
> 
> Is this thread safe?  

It is, because order of main_loop and cpu_exec executions is protected
by global mutex.

> Please introduce the record/replay QemuMutex in
> patch 4, and all along the series document which variables it protects.

record/replay only uses mutex for events queue, because events may be 
added to the queue by the different threads.

Pavel Dovgalyuk
Paolo Bonzini Jan. 13, 2015, 9:38 a.m. UTC | #5
On 13/01/2015 10:21, Pavel Dovgaluk wrote:
>>> +/*! Reads next clock event from the input. */
>>> > > +int64_t replay_read_clock(unsigned int kind)
>>> > > +{
>>> > > +    if (kind >= REPLAY_CLOCK_COUNT) {
>>> > > +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
>>> > > +        exit(1);
>>> > > +    }
>>> > > +
>>> > > +    replay_exec_instructions();
>>> > > +
>>> > > +    if (replay_file) {
>>> > > +        if (skip_async_events(EVENT_CLOCK + kind)) {
>>> > > +            replay_read_next_clock(kind);
>>> > > +        }
>>> > > +        int64_t ret = replay_state.cached_clock[kind];
>>> > > +
>>> > > +        return ret;
>>> > > +    }
>>> > > +
>>> > > +    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
>>> > > +    exit(1);
>>> > > +}
>> > 
>> > Is this thread safe?  
> It is, because order of main_loop and cpu_exec executions is protected
> by global mutex.
> 

Please document exactly which globals are protected by the rr QemuMutex,
and which by the global mutex.  But I think as many variables as
possible should be protected by the rr QemuMutex, for two reasons:

1) people are working on threaded TCG.  While SMP record/replay is a
whole different story, even UP TCG will be multithreaded.

2) in the end it makes reviewing the code simpler.

Paolo
Pavel Dovgalyuk Jan. 16, 2015, 8:03 a.m. UTC | #6
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 13/01/2015 10:21, Pavel Dovgaluk wrote:
> >>> +/*! Reads next clock event from the input. */
> >>> > > +int64_t replay_read_clock(unsigned int kind)
> >>> > > +{
> >>> > > +    if (kind >= REPLAY_CLOCK_COUNT) {
> >>> > > +        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
> >>> > > +        exit(1);
> >>> > > +    }
> >>> > > +
> >>> > > +    replay_exec_instructions();
> >>> > > +
> >>> > > +    if (replay_file) {
> >>> > > +        if (skip_async_events(EVENT_CLOCK + kind)) {
> >>> > > +            replay_read_next_clock(kind);
> >>> > > +        }
> >>> > > +        int64_t ret = replay_state.cached_clock[kind];
> >>> > > +
> >>> > > +        return ret;
> >>> > > +    }
> >>> > > +
> >>> > > +    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
> >>> > > +    exit(1);
> >>> > > +}
> >> >
> >> > Is this thread safe?
> > It is, because order of main_loop and cpu_exec executions is protected
> > by global mutex.
> >
> 
> Please document exactly which globals are protected by the rr QemuMutex,
> and which by the global mutex.  But I think as many variables as
> possible should be protected by the rr QemuMutex, for two reasons:
> 
> 1) people are working on threaded TCG.  While SMP record/replay is a
> whole different story, even UP TCG will be multithreaded.
> 
> 2) in the end it makes reviewing the code simpler.

There is one problem with protecting log file inside the replay code.
We probably should have the following sequence of calls:

lock_replay_mutex
replay_save_events
    replay_run_event
unlock_replay_mutex

But replay_run_event can also generate some log events and we will have deadlock here.
That is why we rely on global mutex.

Pavel Dovgalyuk
Paolo Bonzini Jan. 19, 2015, 11:51 a.m. UTC | #7
On 16/01/2015 09:03, Pavel Dovgaluk wrote:
> There is one problem with protecting log file inside the replay code.
> We probably should have the following sequence of calls:
> 
> lock_replay_mutex
> replay_save_events
>     replay_run_event
> unlock_replay_mutex
> 
> But replay_run_event can also generate some log events and we will have deadlock here.
> That is why we rely on global mutex.

I think replay_run_event should run with the lock _not_ taken, that is:

    qemu_mutex_lock(&lock);
    while (!QTAILQ_EMPTY(&events_list)) {
        Event *event = QTAILQ_FIRST(&events_list);
        QTAILQ_REMOVE(&events_list, event, events);
        qemu_mutex_unlock(&lock);
        replay_run_event(event);
        g_free(event);
        qemu_mutex_lock(&lock);
    }
    qemu_mutex_unlock(&lock);

Paolo
Pavel Dovgalyuk Jan. 19, 2015, 12:03 p.m. UTC | #8
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 16/01/2015 09:03, Pavel Dovgaluk wrote:
> > There is one problem with protecting log file inside the replay code.
> > We probably should have the following sequence of calls:
> >
> > lock_replay_mutex
> > replay_save_events
> >     replay_run_event
> > unlock_replay_mutex
> >
> > But replay_run_event can also generate some log events and we will have deadlock here.
> > That is why we rely on global mutex.
> 
> I think replay_run_event should run with the lock _not_ taken, that is:
> 
>     qemu_mutex_lock(&lock);
>     while (!QTAILQ_EMPTY(&events_list)) {
>         Event *event = QTAILQ_FIRST(&events_list);
>         QTAILQ_REMOVE(&events_list, event, events);
>         qemu_mutex_unlock(&lock);
>         replay_run_event(event);
>         g_free(event);
>         qemu_mutex_lock(&lock);
>     }
>     qemu_mutex_unlock(&lock);

It will work for protecting the events list (I've already did this).
But that will not work for protecting the log file.
replay_run_event can write some data to the log. And also some other function like replay_checkpoint
can also write to the log simultaneously (if we will remove the global mutex).
That's why we cannot simply replace the global mutex with some kind of local one.

Pavel Dovgalyuk
Paolo Bonzini Jan. 19, 2015, 12:15 p.m. UTC | #9
On 19/01/2015 13:03, Pavel Dovgaluk wrote:
> It will work for protecting the events list (I've already did this).
> But that will not work for protecting the log file.
> replay_run_event can write some data to the log. And also some other function like replay_checkpoint
> can also write to the log simultaneously (if we will remove the global mutex).
> That's why we cannot simply replace the global mutex with some kind of local one.

That's why you have to document very well the architecture and the
locking policy.  For example, why can't replay_run_event (or something
that it calls) take the replay lock locally, when it writes to the log?

If you do not document the architecture, we run in circles with me
asking possibly stupid questions and you thinking that my questions are
stupid. :)

Paolo
Pavel Dovgalyuk Jan. 19, 2015, 12:43 p.m. UTC | #10
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 19/01/2015 13:03, Pavel Dovgaluk wrote:
> > It will work for protecting the events list (I've already did this).
> > But that will not work for protecting the log file.
> > replay_run_event can write some data to the log. And also some other function like
> replay_checkpoint
> > can also write to the log simultaneously (if we will remove the global mutex).
> > That's why we cannot simply replace the global mutex with some kind of local one.
> 
> That's why you have to document very well the architecture and the
> locking policy.  

Ok.

> For example, why can't replay_run_event (or something
> that it calls) take the replay lock locally, when it writes to the log?

replay_run_event can take the lock. Suppose that it writes data 'A'.
replay_run_event itself corresponds to some event 'E'.
We expect that the following sequence of the events should occur: 'E', 'A'.
But if something will be written to the log between 'E' and 'A' then
replay_run_event in replay mode will stuck, because it will not see its data 'A'.


Pavel Dovgalyuk
Paolo Bonzini Jan. 19, 2015, 12:57 p.m. UTC | #11
On 19/01/2015 13:43, Pavel Dovgaluk wrote:
> > For example, why can't replay_run_event (or something
> > that it calls) take the replay lock locally, when it writes to the log?
>
> replay_run_event can take the lock. Suppose that it writes data 'A'.
> replay_run_event itself corresponds to some event 'E'.
> We expect that the following sequence of the events should occur: 'E', 'A'.
> But if something will be written to the log between 'E' and 'A' then
> replay_run_event in replay mode will stuck, because it will not see its data 'A'.

It would be easier if you pointed me to actual code in the series.  But
this doesn't seem impossible to fix by atomically writing the 'E' and
'A' in the same critical section.

This is for example how QMP events are thread-safe: it just adds the
JSON string corresponding to the QMP event atomically in monitor_puts.

Paolo
Pavel Dovgalyuk Jan. 19, 2015, 1:01 p.m. UTC | #12
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 19/01/2015 13:43, Pavel Dovgaluk wrote:
> > > For example, why can't replay_run_event (or something
> > > that it calls) take the replay lock locally, when it writes to the log?
> >
> > replay_run_event can take the lock. Suppose that it writes data 'A'.
> > replay_run_event itself corresponds to some event 'E'.
> > We expect that the following sequence of the events should occur: 'E', 'A'.
> > But if something will be written to the log between 'E' and 'A' then
> > replay_run_event in replay mode will stuck, because it will not see its data 'A'.
> 
> It would be easier if you pointed me to actual code in the series.  But
> this doesn't seem impossible to fix by atomically writing the 'E' and
> 'A' in the same critical section.

Because 'A' is written only inside some of the replay_run_event callbacks.
It depends on type of the event and it's processing function inside the QEMU core.
There could be no 'A' at all.

Pavel Dovgalyuk
Paolo Bonzini Jan. 19, 2015, 1:02 p.m. UTC | #13
On 19/01/2015 14:01, Pavel Dovgaluk wrote:
> > It would be easier if you pointed me to actual code in the series.  But
> > this doesn't seem impossible to fix by atomically writing the 'E' and
> > 'A' in the same critical section.
>
> Because 'A' is written only inside some of the replay_run_event callbacks.
> It depends on type of the event and it's processing function inside the QEMU core.
> There could be no 'A' at all.

Why can't that code write the 'E' as well?

Paolo
Pavel Dovgalyuk Jan. 19, 2015, 1:10 p.m. UTC | #14
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 19/01/2015 14:01, Pavel Dovgaluk wrote:
> > > It would be easier if you pointed me to actual code in the series.  But
> > > this doesn't seem impossible to fix by atomically writing the 'E' and
> > > 'A' in the same critical section.
> >
> > Because 'A' is written only inside some of the replay_run_event callbacks.
> > It depends on type of the event and it's processing function inside the QEMU core.
> > There could be no 'A' at all.
> 
> Why can't that code write the 'E' as well?

Because such callbacks do not know that they are called from record/replay event.
They may be called from record/replay code and from other parts of QEMU.
And they may write save something low-level like timer request.

Pavel Dovgalyuk
Paolo Bonzini Jan. 19, 2015, 1:12 p.m. UTC | #15
On 19/01/2015 14:10, Pavel Dovgaluk wrote:
>>> > >
>>> > > Because 'A' is written only inside some of the replay_run_event callbacks.
>>> > > It depends on type of the event and it's processing function inside the QEMU core.
>>> > > There could be no 'A' at all.
>> > 
>> > Why can't that code write the 'E' as well?
> Because such callbacks do not know that they are called from record/replay event.
> They may be called from record/replay code and from other parts of QEMU.
> And they may write save something low-level like timer request.

I do not understand.  Can you make an example with actual function names
and data that is written?

Paolo
Pavel Dovgalyuk Jan. 20, 2015, 6:24 a.m. UTC | #16
> From: Paolo Bonzini [mailto:pbonzini@redhat.com]
> On 19/01/2015 14:10, Pavel Dovgaluk wrote:
> >>> > >
> >>> > > Because 'A' is written only inside some of the replay_run_event callbacks.
> >>> > > It depends on type of the event and it's processing function inside the QEMU core.
> >>> > > There could be no 'A' at all.
> >> >
> >> > Why can't that code write the 'E' as well?
> > Because such callbacks do not know that they are called from record/replay event.
> > They may be called from record/replay code and from other parts of QEMU.
> > And they may write save something low-level like timer request.
> 
> I do not understand.  Can you make an example with actual function names
> and data that is written?

I haven't found any actual examples. Probably they occurred in previous version which wrote
virtual timers into the log.
Anyway I will add the mutex then.

Pavel Dovgalyuk
diff mbox

Patch

diff --git a/cpus.c b/cpus.c
index 8787277..01d89aa 100644
--- a/cpus.c
+++ b/cpus.c
@@ -353,7 +353,8 @@  static void icount_warp_rt(void *opaque)
 
     seqlock_write_lock(&timers_state.vm_clock_seqlock);
     if (runstate_is_running()) {
-        int64_t clock = cpu_get_clock_locked();
+        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
+                                     cpu_get_clock_locked());
         int64_t warp_delta;
 
         warp_delta = clock - vm_clock_warp_start;
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index d9df094..a8bc9eb 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -4,6 +4,7 @@ 
 #include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
+#include "replay/replay.h"
 
 /* timers */
 
@@ -760,6 +761,8 @@  int64_t cpu_icount_to_ns(int64_t icount);
 /*******************************************/
 /* host CPU ticks (if available) */
 
+#define cpu_get_real_ticks cpu_get_real_ticks_impl
+
 #if defined(_ARCH_PPC)
 
 static inline int64_t cpu_get_real_ticks(void)
@@ -913,6 +916,13 @@  static inline int64_t cpu_get_real_ticks (void)
 }
 #endif
 
+#undef cpu_get_real_ticks
+
+static inline int64_t cpu_get_real_ticks(void)
+{
+    return REPLAY_CLOCK(REPLAY_CLOCK_REAL_TICKS, cpu_get_real_ticks_impl());
+}
+
 #ifdef CONFIG_PROFILER
 static inline int64_t profile_getclock(void)
 {
diff --git a/qemu-timer.c b/qemu-timer.c
index cb7d988..19b82f6 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -25,6 +25,7 @@ 
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
 #include "ui/console.h"
+#include "replay/replay.h"
 
 #include "hw/hw.h"
 
@@ -566,15 +567,15 @@  int64_t qemu_clock_get_ns(QEMUClockType type)
             return cpu_get_clock();
         }
     case QEMU_CLOCK_HOST:
-        now = get_clock_realtime();
+        now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
         last = clock->last;
         clock->last = now;
-        if (now < last) {
+        if (now < last && replay_mode == REPLAY_MODE_NONE) {
             notifier_list_notify(&clock->reset_notifiers, &now);
         }
         return now;
     case QEMU_CLOCK_VIRTUAL_RT:
-        return cpu_get_clock();
+        return REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT, cpu_get_clock());
     }
 }
 
diff --git a/replay/Makefile.objs b/replay/Makefile.objs
index 56da09c..257c320 100755
--- a/replay/Makefile.objs
+++ b/replay/Makefile.objs
@@ -1,3 +1,4 @@ 
 obj-y += replay.o
 obj-y += replay-internal.o
 obj-y += replay-events.o
+obj-y += replay-time.o
diff --git a/replay/replay-internal.h b/replay/replay-internal.h
index fcba977..c36d7de 100755
--- a/replay/replay-internal.h
+++ b/replay/replay-internal.h
@@ -22,12 +22,17 @@ 
 #define EVENT_ASYNC_OPT             25
 /* for instruction event */
 #define EVENT_INSTRUCTION           32
+/* for clock read/writes */
+#define EVENT_CLOCK                 64
+/* some of grteater codes are reserved for clocks */
 
 /* Asynchronous events IDs */
 
 #define REPLAY_ASYNC_COUNT             0
 
 typedef struct ReplayState {
+    /*! Cached clock values. */
+    int64_t cached_clock[REPLAY_CLOCK_COUNT];
     /*! Current step - number of processed instructions and timer events. */
     uint64_t current_step;
     /*! Number of instructions to be executed before other events happen. */
@@ -75,6 +80,12 @@  bool skip_async_events(int stop_event);
     reports an error and stops the execution. */
 void skip_async_events_until(unsigned int kind);
 
+/*! Reads next clock value from the file.
+    If clock kind read from the file is different from the parameter,
+    the value is not used.
+    If the parameter is -1, the clock value is read to the cache anyway. */
+void replay_read_next_clock(unsigned int kind);
+
 /* Asynchronous events queue */
 
 /*! Initializes events' processing internals */
diff --git a/replay/replay-time.c b/replay/replay-time.c
new file mode 100755
index 0000000..3f94f4e
--- /dev/null
+++ b/replay/replay-time.c
@@ -0,0 +1,79 @@ 
+/*
+ * replay-time.c
+ *
+ * Copyright (c) 2010-2014 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu-common.h"
+#include "replay.h"
+#include "replay-internal.h"
+
+
+int64_t replay_save_clock(unsigned int kind, int64_t clock)
+{
+    replay_save_instructions();
+
+    if (kind >= REPLAY_CLOCK_COUNT) {
+        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
+        exit(1);
+    }
+
+    if (replay_file) {
+        replay_put_event(EVENT_CLOCK + kind);
+        replay_put_qword(clock);
+    }
+
+    return clock;
+}
+
+void replay_read_next_clock(unsigned int kind)
+{
+    replay_fetch_data_kind();
+    if (replay_file) {
+        unsigned int read_kind = replay_data_kind - EVENT_CLOCK;
+
+        if (kind != -1 && read_kind != kind) {
+            return;
+        }
+        if (read_kind >= REPLAY_CLOCK_COUNT) {
+            fprintf(stderr,
+                    "invalid clock ID %d was read from replay\n", read_kind);
+            exit(1);
+        }
+
+        int64_t clock = replay_get_qword();
+
+        replay_check_error();
+        replay_has_unread_data = 0;
+
+        replay_state.cached_clock[read_kind] = clock;
+    }
+}
+
+/*! Reads next clock event from the input. */
+int64_t replay_read_clock(unsigned int kind)
+{
+    if (kind >= REPLAY_CLOCK_COUNT) {
+        fprintf(stderr, "invalid clock ID %d for replay\n", kind);
+        exit(1);
+    }
+
+    replay_exec_instructions();
+
+    if (replay_file) {
+        if (skip_async_events(EVENT_CLOCK + kind)) {
+            replay_read_next_clock(kind);
+        }
+        int64_t ret = replay_state.cached_clock[kind];
+
+        return ret;
+    }
+
+    fprintf(stderr, "REPLAY INTERNAL ERROR %d\n", __LINE__);
+    exit(1);
+}
diff --git a/replay/replay.h b/replay/replay.h
index 90a949b..1c18c0e 100755
--- a/replay/replay.h
+++ b/replay/replay.h
@@ -16,6 +16,16 @@ 
 #include <stdint.h>
 #include "qapi-types.h"
 
+/* replay clock kinds */
+/* rdtsc */
+#define REPLAY_CLOCK_REAL_TICKS 0
+/* host_clock */
+#define REPLAY_CLOCK_HOST       1
+/* virtual_rt_clock */
+#define REPLAY_CLOCK_VIRTUAL_RT 2
+
+#define REPLAY_CLOCK_COUNT      3
+
 extern ReplayMode replay_mode;
 extern char *replay_image_suffix;
 
@@ -47,6 +57,19 @@  bool replay_interrupt(void);
     Returns true, when interrupt request is pending */
 bool replay_has_interrupt(void);
 
+/* Processing clocks and other time sources */
+
+/*! Save the specified clock */
+int64_t replay_save_clock(unsigned int kind, int64_t clock);
+/*! Read the specified clock from the log or return cached data */
+int64_t replay_read_clock(unsigned int kind);
+/*! Saves or reads the clock depending on the current replay mode. */
+#define REPLAY_CLOCK(clock, value)                                      \
+    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
+        : replay_mode == REPLAY_MODE_RECORD                             \
+            ? replay_save_clock((clock), (value))                       \
+        : (value))
+
 /* Asynchronous events queue */
 
 /*! Disables storing events in the queue */
diff --git a/stubs/replay.c b/stubs/replay.c
index f923d7d..397d4f2 100755
--- a/stubs/replay.c
+++ b/stubs/replay.c
@@ -6,3 +6,12 @@  ReplaySubmode replay_get_play_submode(void)
 {
     return REPLAY_SUBMODE_UNKNOWN;
 }
+
+int64_t replay_save_clock(unsigned int kind, int64_t clock)
+{
+}
+
+int64_t replay_read_clock(unsigned int kind)
+{
+    return 0;
+}