[PATCH] sched: avoid unnecessary overflow in sched_clock

[PATCH] sched: avoid unnecessary overflow in sched_clock

[Salman Qazi]

In hundreds of days, the __cycles_2_ns calculation in sched_clock
has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
the final value to become zero.  We can solve this without losing
any precision.

We can decompose TSC into quotient and remainder of division by the
scale factor, and then use this to convert TSC into nanoseconds.
---
 arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
 1 files changed, 22 insertions(+), 1 deletions(-)

diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index fa7b917..431793e 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -32,6 +32,22 @@ extern int no_timer_check;
  *  (mathieu.desnoyers@polymtl.ca)
  *
  *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
+ *
+ * In:
+ *
+ * ns = cycles * cyc2ns_scale / SC
+ *
+ * Although we may still have enough bits to store the value of ns,
+ * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
+ * leading to an incorrect result.
+ *
+ * To avoid this, we can decompose 'cycles' into quotient and remainder
+ * of division by SC.  Then,
+ *
+ * ns = (quot * SC + rem) * cyc2ns_scale / SC
+ *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
+ *
+ *			- sqazi@google.com
  */
 
 DECLARE_PER_CPU(unsigned long, cyc2ns);
@@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
 
 static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
 {
+	unsigned long long quot;
+	unsigned long long rem;
 	int cpu = smp_processor_id();
 	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
-	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
+	quot = (cyc >> CYC2NS_SCALE_FACTOR);
+	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
+	ns += quot * per_cpu(cyc2ns, cpu) +
+		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
 	return ns;
 }
 

[PATCH] sched: avoid unnecessary overflow in sched_clock

[Salman Qazi]

(Added the missing signed-off-by line)

In hundreds of days, the __cycles_2_ns calculation in sched_clock
has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
the final value to become zero.  We can solve this without losing
any precision.

We can decompose TSC into quotient and remainder of division by the
scale factor, and then use this to convert TSC into nanoseconds.

Signed-off-by: Salman Qazi <sqazi@google.com>
---
 arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
 1 files changed, 22 insertions(+), 1 deletions(-)

diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index fa7b917..431793e 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -32,6 +32,22 @@ extern int no_timer_check;
  *  (mathieu.desnoyers@polymtl.ca)
  *
  *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
+ *
+ * In:
+ *
+ * ns = cycles * cyc2ns_scale / SC
+ *
+ * Although we may still have enough bits to store the value of ns,
+ * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
+ * leading to an incorrect result.
+ *
+ * To avoid this, we can decompose 'cycles' into quotient and remainder
+ * of division by SC.  Then,
+ *
+ * ns = (quot * SC + rem) * cyc2ns_scale / SC
+ *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
+ *
+ *			- sqazi@google.com
  */
 
 DECLARE_PER_CPU(unsigned long, cyc2ns);
@@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
 
 static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
 {
+	unsigned long long quot;
+	unsigned long long rem;
 	int cpu = smp_processor_id();
 	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
-	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
+	quot = (cyc >> CYC2NS_SCALE_FACTOR);
+	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
+	ns += quot * per_cpu(cyc2ns, cpu) +
+		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
 	return ns;
 }
 

Re: [PATCH] sched: avoid unnecessary overflow in sched_clock

[john stultz]

On Tue, 2011-11-15 at 14:12 -0800, Salman Qazi wrote:
> (Added the missing signed-off-by line)
> 
> In hundreds of days, the __cycles_2_ns calculation in sched_clock
> has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
> the final value to become zero.  We can solve this without losing
> any precision.
> 
> We can decompose TSC into quotient and remainder of division by the
> scale factor, and then use this to convert TSC into nanoseconds.
> 
> Signed-off-by: Salman Qazi <sqazi@google.com>

Acked-by: John Stultz <johnstul@us.ibm.com>


> ---
>  arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
>  1 files changed, 22 insertions(+), 1 deletions(-)
> 
> diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
> index fa7b917..431793e 100644
> --- a/arch/x86/include/asm/timer.h
> +++ b/arch/x86/include/asm/timer.h
> @@ -32,6 +32,22 @@ extern int no_timer_check;
>   *  (mathieu.desnoyers@polymtl.ca)
>   *
>   *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
> + *
> + * In:
> + *
> + * ns = cycles * cyc2ns_scale / SC
> + *
> + * Although we may still have enough bits to store the value of ns,
> + * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
> + * leading to an incorrect result.
> + *
> + * To avoid this, we can decompose 'cycles' into quotient and remainder
> + * of division by SC.  Then,
> + *
> + * ns = (quot * SC + rem) * cyc2ns_scale / SC
> + *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
> + *
> + *			- sqazi@google.com
>   */
> 
>  DECLARE_PER_CPU(unsigned long, cyc2ns);
> @@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
> 
>  static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
>  {
> +	unsigned long long quot;
> +	unsigned long long rem;
>  	int cpu = smp_processor_id();
>  	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
> -	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
> +	quot = (cyc >> CYC2NS_SCALE_FACTOR);
> +	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
> +	ns += quot * per_cpu(cyc2ns, cpu) +
> +		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
>  	return ns;
>  }
> 
> 

Re: [PATCH] sched: avoid unnecessary overflow in sched_clock

[Paul Turner]

On 11/15/2011 03:02 PM, john stultz wrote:
> On Tue, 2011-11-15 at 14:12 -0800, Salman Qazi wrote:
>> (Added the missing signed-off-by line)
>>
>> In hundreds of days, the __cycles_2_ns calculation in sched_clock
>> has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
>> the final value to become zero.  We can solve this without losing
>> any precision.
>>
>> We can decompose TSC into quotient and remainder of division by the
>> scale factor, and then use this to convert TSC into nanoseconds.
>>
>> Signed-off-by: Salman Qazi<sqazi@google.com>
>
> Acked-by: John Stultz<johnstul@us.ibm.com>
>

Reviewed-by: Paul Turner <pjt@google.com>

>
>> ---
>>   arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
>>   1 files changed, 22 insertions(+), 1 deletions(-)
>>
>> diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
>> index fa7b917..431793e 100644
>> --- a/arch/x86/include/asm/timer.h
>> +++ b/arch/x86/include/asm/timer.h
>> @@ -32,6 +32,22 @@ extern int no_timer_check;
>>    *  (mathieu.desnoyers@polymtl.ca)
>>    *
>>    *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
>> + *
>> + * In:
>> + *
>> + * ns = cycles * cyc2ns_scale / SC
>> + *
>> + * Although we may still have enough bits to store the value of ns,
>> + * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
>> + * leading to an incorrect result.
>> + *
>> + * To avoid this, we can decompose 'cycles' into quotient and remainder
>> + * of division by SC.  Then,
>> + *
>> + * ns = (quot * SC + rem) * cyc2ns_scale / SC
>> + *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
>> + *
>> + *			- sqazi@google.com
>>    */
>>
>>   DECLARE_PER_CPU(unsigned long, cyc2ns);
>> @@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
>>
>>   static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
>>   {
>> +	unsigned long long quot;
>> +	unsigned long long rem;
>>   	int cpu = smp_processor_id();
>>   	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
>> -	ns += cyc * per_cpu(cyc2ns, cpu)>>  CYC2NS_SCALE_FACTOR;
>> +	quot = (cyc>>  CYC2NS_SCALE_FACTOR);
>> +	rem = cyc&  ((1ULL<<  CYC2NS_SCALE_FACTOR) - 1);
>> +	ns += quot * per_cpu(cyc2ns, cpu) +
>> +		((rem * per_cpu(cyc2ns, cpu))>>  CYC2NS_SCALE_FACTOR);
>>   	return ns;
>>   }
>>
>>
>
>

Re: [PATCH] sched: avoid unnecessary overflow in sched_clock

[Mike Galbraith]

On Tue, 2011-11-15 at 15:02 -0800, john stultz wrote:
> On Tue, 2011-11-15 at 14:12 -0800, Salman Qazi wrote:
> > (Added the missing signed-off-by line)
> > 
> > In hundreds of days, the __cycles_2_ns calculation in sched_clock
> > has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
> > the final value to become zero.  We can solve this without losing
> > any precision.
> > 
> > We can decompose TSC into quotient and remainder of division by the
> > scale factor, and then use this to convert TSC into nanoseconds.
> > 
> > Signed-off-by: Salman Qazi <sqazi@google.com>
> 
> Acked-by: John Stultz <johnstul@us.ibm.com>

This wants a stable tag, no?

	-Mike

> > ---
> >  arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
> >  1 files changed, 22 insertions(+), 1 deletions(-)
> > 
> > diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
> > index fa7b917..431793e 100644
> > --- a/arch/x86/include/asm/timer.h
> > +++ b/arch/x86/include/asm/timer.h
> > @@ -32,6 +32,22 @@ extern int no_timer_check;
> >   *  (mathieu.desnoyers@polymtl.ca)
> >   *
> >   *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
> > + *
> > + * In:
> > + *
> > + * ns = cycles * cyc2ns_scale / SC
> > + *
> > + * Although we may still have enough bits to store the value of ns,
> > + * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
> > + * leading to an incorrect result.
> > + *
> > + * To avoid this, we can decompose 'cycles' into quotient and remainder
> > + * of division by SC.  Then,
> > + *
> > + * ns = (quot * SC + rem) * cyc2ns_scale / SC
> > + *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
> > + *
> > + *			- sqazi@google.com
> >   */
> > 
> >  DECLARE_PER_CPU(unsigned long, cyc2ns);
> > @@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
> > 
> >  static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
> >  {
> > +	unsigned long long quot;
> > +	unsigned long long rem;
> >  	int cpu = smp_processor_id();
> >  	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
> > -	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
> > +	quot = (cyc >> CYC2NS_SCALE_FACTOR);
> > +	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
> > +	ns += quot * per_cpu(cyc2ns, cpu) +
> > +		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
> >  	return ns;
> >  }
> > 
> > 
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH] sched: avoid unnecessary overflow in sched_clock

[Paul Turner]

On 11/15/2011 10:41 PM, Mike Galbraith wrote:
> On Tue, 2011-11-15 at 15:02 -0800, john stultz wrote:
>> On Tue, 2011-11-15 at 14:12 -0800, Salman Qazi wrote:
>>> (Added the missing signed-off-by line)
>>>
>>> In hundreds of days, the __cycles_2_ns calculation in sched_clock
>>> has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
>>> the final value to become zero.  We can solve this without losing
>>> any precision.
>>>
>>> We can decompose TSC into quotient and remainder of division by the
>>> scale factor, and then use this to convert TSC into nanoseconds.
>>>
>>> Signed-off-by: Salman Qazi<sqazi@google.com>
>>
>> Acked-by: John Stultz<johnstul@us.ibm.com>
>
> This wants a stable tag, no?
>
> 	-Mike
>

Probably a good idea -- This especially sucks rocks in the sched_clock_stable==1 
case; resulting in it coming straight back out of sched_clock_cpu() and trashing 
rq->clock.

- Paul

>>> ---
>>>   arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
>>>   1 files changed, 22 insertions(+), 1 deletions(-)
>>>
>>> diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
>>> index fa7b917..431793e 100644
>>> --- a/arch/x86/include/asm/timer.h
>>> +++ b/arch/x86/include/asm/timer.h
>>> @@ -32,6 +32,22 @@ extern int no_timer_check;
>>>    *  (mathieu.desnoyers@polymtl.ca)
>>>    *
>>>    *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
>>> + *
>>> + * In:
>>> + *
>>> + * ns = cycles * cyc2ns_scale / SC
>>> + *
>>> + * Although we may still have enough bits to store the value of ns,
>>> + * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
>>> + * leading to an incorrect result.
>>> + *
>>> + * To avoid this, we can decompose 'cycles' into quotient and remainder
>>> + * of division by SC.  Then,
>>> + *
>>> + * ns = (quot * SC + rem) * cyc2ns_scale / SC
>>> + *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
>>> + *
>>> + *			- sqazi@google.com
>>>    */
>>>
>>>   DECLARE_PER_CPU(unsigned long, cyc2ns);
>>> @@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
>>>
>>>   static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
>>>   {
>>> +	unsigned long long quot;
>>> +	unsigned long long rem;
>>>   	int cpu = smp_processor_id();
>>>   	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
>>> -	ns += cyc * per_cpu(cyc2ns, cpu)>>  CYC2NS_SCALE_FACTOR;
>>> +	quot = (cyc>>  CYC2NS_SCALE_FACTOR);
>>> +	rem = cyc&  ((1ULL<<  CYC2NS_SCALE_FACTOR) - 1);
>>> +	ns += quot * per_cpu(cyc2ns, cpu) +
>>> +		((rem * per_cpu(cyc2ns, cpu))>>  CYC2NS_SCALE_FACTOR);
>>>   	return ns;
>>>   }
>>>
>>>
>>
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> Please read the FAQ at  http://www.tux.org/lkml/
>
>

Re: [PATCH] sched: avoid unnecessary overflow in sched_clock

[Peter Zijlstra]

On Tue, 2011-11-15 at 23:08 -0800, Paul Turner wrote:
> Probably a good idea -- This especially sucks rocks in the sched_clock_stable==1 
> case; resulting in it coming straight back out of sched_clock_cpu() and trashing 
> rq->clock. 

There's another problem there in that TSC will wrap too soon and things
will blow up in your face that way.

[PATCH] sched: avoid unnecessary overflow in sched_clock

[Salman Qazi]

(Added Reviewed-by/Acked-by tags)

In hundreds of days, the __cycles_2_ns calculation in sched_clock
has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
the final value to become zero.  We can solve this without losing
any precision.

We can decompose TSC into quotient and remainder of division by the
scale factor, and then use this to convert TSC into nanoseconds.

Reviewed-by: Paul Turner <pjt@google.com>
Acked-by: John Stultz<johnstul@us.ibm.com>
Signed-off-by: Salman Qazi <sqazi@google.com>
---
 arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
 1 files changed, 22 insertions(+), 1 deletions(-)

diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index fa7b917..431793e 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -32,6 +32,22 @@ extern int no_timer_check;
  *  (mathieu.desnoyers@polymtl.ca)
  *
  *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
+ *
+ * In:
+ *
+ * ns = cycles * cyc2ns_scale / SC
+ *
+ * Although we may still have enough bits to store the value of ns,
+ * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
+ * leading to an incorrect result.
+ *
+ * To avoid this, we can decompose 'cycles' into quotient and remainder
+ * of division by SC.  Then,
+ *
+ * ns = (quot * SC + rem) * cyc2ns_scale / SC
+ *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
+ *
+ *			- sqazi@google.com
  */
 
 DECLARE_PER_CPU(unsigned long, cyc2ns);
@@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
 
 static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
 {
+	unsigned long long quot;
+	unsigned long long rem;
 	int cpu = smp_processor_id();
 	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
-	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
+	quot = (cyc >> CYC2NS_SCALE_FACTOR);
+	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
+	ns += quot * per_cpu(cyc2ns, cpu) +
+		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
 	return ns;
 }
 

[tip:sched/urgent] sched, x86: Avoid unnecessary overflow in sched_clock

[tip-bot for Salman Qazi]

Commit-ID:  4cecf6d401a01d054afc1e5f605bcbfe553cb9b9
Gitweb:     http://git.kernel.org/tip/4cecf6d401a01d054afc1e5f605bcbfe553cb9b9
Author:     Salman Qazi <sqazi@google.com>
AuthorDate: Tue, 15 Nov 2011 14:12:06 -0800
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Wed, 16 Nov 2011 19:51:25 +0100

sched, x86: Avoid unnecessary overflow in sched_clock

(Added the missing signed-off-by line)

In hundreds of days, the __cycles_2_ns calculation in sched_clock
has an overflow.  cyc * per_cpu(cyc2ns, cpu) exceeds 64 bits, causing
the final value to become zero.  We can solve this without losing
any precision.

We can decompose TSC into quotient and remainder of division by the
scale factor, and then use this to convert TSC into nanoseconds.

Signed-off-by: Salman Qazi <sqazi@google.com>
Acked-by: John Stultz <johnstul@us.ibm.com>
Reviewed-by: Paul Turner <pjt@google.com>
Cc: stable@kernel.org
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20111115221121.7262.88871.stgit@dungbeetle.mtv.corp.google.com
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 arch/x86/include/asm/timer.h |   23 ++++++++++++++++++++++-
 1 files changed, 22 insertions(+), 1 deletions(-)

diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h
index fa7b917..431793e 100644
--- a/arch/x86/include/asm/timer.h
+++ b/arch/x86/include/asm/timer.h
@@ -32,6 +32,22 @@ extern int no_timer_check;
  *  (mathieu.desnoyers@polymtl.ca)
  *
  *			-johnstul@us.ibm.com "math is hard, lets go shopping!"
+ *
+ * In:
+ *
+ * ns = cycles * cyc2ns_scale / SC
+ *
+ * Although we may still have enough bits to store the value of ns,
+ * in some cases, we may not have enough bits to store cycles * cyc2ns_scale,
+ * leading to an incorrect result.
+ *
+ * To avoid this, we can decompose 'cycles' into quotient and remainder
+ * of division by SC.  Then,
+ *
+ * ns = (quot * SC + rem) * cyc2ns_scale / SC
+ *    = quot * cyc2ns_scale + (rem * cyc2ns_scale) / SC
+ *
+ *			- sqazi@google.com
  */
 
 DECLARE_PER_CPU(unsigned long, cyc2ns);
@@ -41,9 +57,14 @@ DECLARE_PER_CPU(unsigned long long, cyc2ns_offset);
 
 static inline unsigned long long __cycles_2_ns(unsigned long long cyc)
 {
+	unsigned long long quot;
+	unsigned long long rem;
 	int cpu = smp_processor_id();
 	unsigned long long ns = per_cpu(cyc2ns_offset, cpu);
-	ns += cyc * per_cpu(cyc2ns, cpu) >> CYC2NS_SCALE_FACTOR;
+	quot = (cyc >> CYC2NS_SCALE_FACTOR);
+	rem = cyc & ((1ULL << CYC2NS_SCALE_FACTOR) - 1);
+	ns += quot * per_cpu(cyc2ns, cpu) +
+		((rem * per_cpu(cyc2ns, cpu)) >> CYC2NS_SCALE_FACTOR);
 	return ns;
 }