# BigDecimal#to\_f precision issue

**URL:** <https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523>\
**Category:** Development\
**Created:** [February 8, 2008, 12:25pm UTC](https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523 "2008-02-08T12:25:34Z")\
**Posts on this page:** 4\
**Page:** 1

<div class="post-metadata">

**Author:** ![nobu](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/n/e9c0ed/32.png) [@nobu](https://www.ruby-forum.com/u/nobu)\
**Post date:** [February 8, 2008, 12:25pm UTC](https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523/1 "2008-02-08T12:25:34Z")

</div>

e$B$J$+$@$G$9!#e(B

[ruby-talk:290296]e$B$NOC$G$9$,!"e(BBigDecimal#to\_fe$B$N@:EY$,IT==J,$J$h$&$G$9!#e(B

$ ruby18 -v -rbigdecimal -e ‘printf “%.15f\n”,  
BigDecimal.new(“15.25”).to\_f’  
ruby 1.8.6 (2008-02-08 revision 15402) [i686-linux]  
15.250000000000002  
$ ruby19 -v -rbigdecimal -e ‘printf “%.15f\n”,  
BigDecimal.new(“15.25”).to\_f’  
ruby 1.9.0 (2008-02-08 revision 15402) [i686-linux]  
15.250000000000002

e$BMW$9$k$Ke(B10e$B?J"\*e(B2e$B?JJQ49$NLdBj$J$N$G!"e(Bstrtod()e$B$+$i;}$C$F$3$h$&$+$He(B  
e$B$b;W$$$^$7$?$,!"e(Bstrtod()e$B$r$=$N$^$^;H$&$N$,0lHV4JC1$8$c$J$$$G$7$ge(B  
e$B$&$+!#e(B

e$B$^$?!“e(BDBLE\_FIGe$B$rFbIt$G7W;;$7$F$$$^$9$,!”%l%8%9%?4V$N1i;;$K:GE,2=e(B  
e$B$5$l$?\>l9g!“e(Bdoublee$B$G$O$J$/$Fe(Blong  
doublee$B$G7W;;$5$l$F$7$^$&\>l9g$,e(B  
e$B$”$j$^$9!#e(BDBL\_DIGe$B$r;H$&$Y$-$G$7$g$&!#e(B

e$B$b$&0lE@!"e(BBASEe$B$de(BBASE\_FIGe$B$b\<B9T;~$K5a$a$k$h$j$bDj?t$K$7$?$[$&$,$$e(B  
e$B$$$N$G$O$J$$$+$H;W$$$^$9!#e(B

# Index: ext/bigdecimal/bigdecimal.c

— ext/bigdecimal/bigdecimal.c (revision 15405)  
+++ ext/bigdecimal/bigdecimal.c (working copy)  
@@ -586,17 +586,20 @@ BigDecimal\_to\_f(VALUE self)  
ENTER(1);  
Real \*p;

- double d, d2;

- double d;  
S\_LONG e;

- char \*buf;

- buf = ALLOCA\_N(char,(unsigned int)VpNumOfChars(p,“E”));

- VpToString(p, buf, 0, 0);  
errno = 0;

- d2 = pow(10.0,(double)e);
- if((errno == ERANGE && e\>0) || (d2\>1.0 && (fabs(d) \> (DBL\_MAX /  
d2)))) {

- d = strtod(buf, 0);
- if(errno == ERANGE) {  
VpException(VP\_EXCEPTION\_OVERFLOW,“BigDecimal to Float  
conversion”,0);  
if(d\>0.0) return rb\_float\_new(DBL\_MAX);  
else return rb\_float\_new(-DBL\_MAX);  
}

- return rb\_float\_new(d\*d2);

- return rb\_float\_new(d);  
}

@@ -1051,5 +1054,5 @@ BigDecimal\_remainder(VALUE self, VALUE r  
{  
VALUE f;

- Real \*d,\*rv;

- Real \*d,\*rv=0;  
f = BigDecimal\_divremain(self,r,&d,&rv);  
if(f!=(VALUE)0) return f;  
@@ -1505,5 +1508,5 @@ BigDecimal\_to\_s(int argc, VALUE \*argv, V  
fPlus = 2; psz++;  
}

- 

```
       while(ch=*psz++) {

```

- 

```
       while((ch=*psz++)!=0) {
           if(ISSPACE(ch)) continue;
           if(!ISDIGIT(ch)) {

```

@@ -1962,4 +1965,5 @@ static U\_LONG gnPrecLimit = 0; /\* Globa  
static U\_LONG gfRoundMode = VP\_ROUND\_HALF\_UP; /\* Mode for general  
rounding operation \*/

+#ifndef BASE\_FIG  
static U\_LONG BASE\_FIG = 4; /\* =log10(BASE) _/  
static U\_LONG BASE = 10000L; /_ Base value(value must be  
10\*\*BASE\_FIG) _/  
@@ -1967,10 +1971,19 @@ static U\_LONG BASE = 10000L; /_ Base  
/\* within one U\_LONG. _/  
static U\_LONG HALF\_BASE = 5000L;/_ =BASE/2 _/  
-static S\_LONG DBLE\_FIG = 8; /_ figure of double _/  
static U\_LONG BASE1 = 1000L; /_ =BASE/10 _/  
+#else  
+#ifndef BASE  
+#error BASE\_FIG is defined but BASE is not  
+#endif  
+#define HALF\_BASE (BASE/2)  
+#define BASE1 (BASE/10)  
+#endif  
+#ifndef DBLE\_FIG  
+#define DBLE\_FIG (DBL\_DIG+1) /_ figure of double \*/  
+#endif

static Real _VpConstOne; /_ constant 1.0 \*/  
static Real _VpPt5; /_ constant 0.5 _/  
-static U\_LONG maxnr = 100; /_ Maximum iterations for calcurating  
sqrt. _/  
+#define maxnr 100UL /_ Maximum iterations for calcurating sqrt. _/  
/_ used in VpSqrt() \*/

@@ -2089,6 +2102,6 @@ VpSetRoundMode(unsigned long n)

- (… but not actually…)).  
\*/  
-volatile double gZero\_ABCED9B1\_CE73\_\_00400511F31D = 0.0;  
-volatile double gOne\_ABCED9B4\_CE73\_\_00400511F31D = 1.0;  
+volatile const double gZero\_ABCED9B1\_CE73\_\_00400511F31D = 0.0;  
+volatile const double gOne\_ABCED9B4\_CE73\_\_00400511F31D = 1.0;  
static double  
Zero(void)  
@@ -2362,7 +2375,4 @@ VP\_EXPORT U\_LONG  
VpInit(U\_LONG BaseVal)  
{

- U\_LONG w;
- double v;
- /\* Setup +/- Inf NaN -0 \*/  
VpGetDoubleNaN();  
@@ -2371,5 +2381,7 @@ VpInit(U\_LONG BaseVal)  
VpGetDoubleNegZero();

+#ifndef BASE\_FIG  
if(BaseVal \<= 0) {

- 

```
   U_LONG w;
   /* Base <= 0, then determine Base by calcuration. */
   BASE = 1;

```

@@ -2388,4 +2400,6 @@ VpInit(U\_LONG BaseVal)  
BASE\_FIG = 0;  
while(BaseVal /= 10) ++BASE\_FIG;  
+#endif  
+  
/\* Allocates Vp constants. _/  
VpConstOne = VpAlloc((U\_LONG)1, “1”);  
@@ -2396,13 +2410,4 @@ VpInit(U\_LONG BaseVal)  
 #endif /_ \_DEBUG \*/

- /\* Determine # of digits available in one ‘double’. \*/
- 
- v = 1.0;
- DBLE\_FIG = 0;
- while(v + 1.0 \> 1.0) {
- 

```
   ++DBLE_FIG;

```

- 

```
   v /= 10;

```

- }
- 

#ifdef \_DEBUG  
if(gfDebug) {  
@@ -2507,5 +2512,5 @@ VpAlloc(U\_LONG mx, const char \*szVal)  
i = 0;  
ipn = 0;

- while(psz[i]=szVal[ipn]) {

- while((psz[i]=szVal[ipn])!=0) {  
if(ISDIGIT(psz[i])) ++ni;  
if(psz[i]==’\_’) {  
@@ -2550,5 +2555,5 @@ VpAlloc(U\_LONG mx, const char _szVal)  
/_ Skip digits _/  
ni = 0; /_ digits in mantissa \*/

- while(v = szVal[i]) {

- while((v = szVal[i]) != 0) {  
if(!ISDIGIT(v)) break;  
++i;  
@@ -2564,5 +2569,5 @@ VpAlloc(U\_LONG mx, const char \*szVal)  
++i;  
ipf = i;

- 

```
       while(v = szVal[i]) { /* get fraction part. */

```

- 

```
       while((v = szVal[i]) != 0) { /* get fraction part. */
           if(!ISDIGIT(v)) break;
           ++i;

```

@@ -2582,5 +2587,5 @@ VpAlloc(U\_LONG mx, const char \*szVal)  
v = szVal[i];  
if((v == ‘-’) ||(v == ‘+’)) ++i;

- 

```
       while(v=szVal[i]) {

```

- 

```
       while((v=szVal[i])!=0) {
           if(!ISDIGIT(v)) break;
           ++i;

```

@@ -4440,5 +4445,5 @@ VpLeftRound(Real \*y, int f, int nf)  
v = y-\>frac[0];  
nf -= VpExponent(y)\*BASE\_FIG;

- while(v=v/10) nf–;

- while((v /= 10) != 0) nf–;  
nf += (BASE\_FIG-1);  
return VpMidRound(y,f,nf);  
Index: ext/bigdecimal/extconf.rb  
===================================================================  
— ext/bigdecimal/extconf.rb (revision 15405)  
+++ ext/bigdecimal/extconf.rb (working copy)  
@@ -1,2 +1,12 @@  
require ‘mkmf’
- 

+base\_fig = 0  
+src = ("(BASE \> 0) && "

- 

```
  "(BASE * (BASE+1)) > BASE && "

```

- 

```
  "(BASE * (BASE+1)) / BASE == (BASE+1)")

```

+while try\_static\_assert(src, nil, “-DBASE=10#{‘0’\*base\_fig}UL”)

- base\_fig += 1  
+end  
+$defs \<\< “-DBASE=1#{‘0’\*base\_fig}UL” \<\< “-DBASE\_FIG=#{base\_fig}”
- 

create\_makefile(‘bigdecimal’)

---

<div class="post-metadata">

**Author:** ![nobu](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/n/e9c0ed/32.png) [@nobu](https://www.ruby-forum.com/u/nobu)\
**Post date:** [February 9, 2008, 3:06am UTC](https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523/2 "2008-02-09T03:06:54Z")

</div>

e$B$^$D$b$He(B e$B$f$-$R$m$G$9e(B

In message “Re: [ruby-dev:33658] BigDecimal#to\_f precision issue”  
on Fri, 8 Feb 2008 20:24:29 +0900, Nobuyoshi N.  
[removed\_email\_address@domain.invalid](mailto:removed_email_address@domain.invalid) writes:  
|  
|e$B$J$+$@$G$9!#e(B  
|  
|[ruby-talk:290296]e$B$NOC$G$9$,!“e(BBigDecimal#to\_fe$B$N@:EY$,IT==J,$J$h$&$G$9!#e(B  
|  
| $ ruby18 -v -rbigdecimal -e ‘printf “%.15f\n”, BigDecimal.new(“15.25”).to\_f’  
| ruby 1.8.6 (2008-02-08 revision 15402) [i686-linux]  
| 15.250000000000002  
| $ ruby19 -v -rbigdecimal -e ‘printf “%.15f\n”, BigDecimal.new(“15.25”).to\_f’  
| ruby 1.9.0 (2008-02-08 revision 15402) [i686-linux]  
| 15.250000000000002  
|  
|e$BMW$9$k$Ke(B10e$B?J”\*e(B2e$B?JJQ49$NLdBj$J$N$G!"e(Bstrtod()e$B$+$i;}$C$F$3$h$&$+$He(B  
|e$B$b;W$$$^$7$?$,!"e(Bstrtod()e$B$r$=$N$^$^;H$&$N$,0lHV4JC1$8$c$J$$$G$7$ge(B  
|e$B$&$+!#e(B  
|  
|e$B$^$?!“e(BDBLE\_FIGe$B$rFbIt$G7W;;$7$F$$$^$9$,!”%l%8%9%?4V$N1i;;$K:GE,2=e(B  
|e$B$5$l$?\>l9g!“e(Bdoublee$B$G$O$J$/$Fe(Blong doublee$B$G7W;;$5$l$F$7$^$&\>l9g$,e(B  
|e$B$”$j$^$9!#e(BDBL\_DIGe$B$r;H$&$Y$-$G$7$g$&!#e(B

e$B%3%\_%C%H$7$F$/$@$5$$!#e(B

|e$B$b$&0lE@!"e(BBASEe$B$de(BBASE\_FIGe$B$b\<B9T;~$K5a$a$k$h$j$bDj?t$K$7$?$[$&$,$$e(B  
|e$B$$$N$G$O$J$$$+$H;W$$$^$9!#e(B

e$B$3$3$O%/%m%9%3%s%Q%$%k$N$3$H$O9M$($J$/$F$h$+$C$?$s$G$9$C$1!#e(B

---

<div class="post-metadata">

**Author:** ![nobu](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/n/e9c0ed/32.png) [@nobu](https://www.ruby-forum.com/u/nobu)\
**Post date:** [February 9, 2008, 9:49am UTC](https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523/3 "2008-02-09T09:49:33Z")

</div>

e$B$J$+$@$G$9!#e(B

At Sat, 9 Feb 2008 11:05:17 +0900,  
Yukihiro M. wrote in [ruby-dev:33665]:

> |e$B$b$&0lE@!"e(BBASEe$B$de(BBASE\_FIGe$B$b\<B9T;~$K5a$a$k$h$j$bDj?t$K$7$?$[$&$,$$e(B  
> |e$B$$$N$G$O$J$$$+$H;W$$$^$9!#e(B
> 
> e$B$3$3$O%/%m%9%3%s%Q%$%k$N$3$H$O9M$($J$/$F$h$+$C$?$s$G$9$C$1!#e(B

try\_static\_asserte$B$O!"$"$k\<0$,%3%s%Q%$%k;~$K??$KI\>2A$5$l$k$+$rD4e(B  
e$B$Y$k$b$N$J$N$G!"%/%m%9%3%s%Q%$%k$G$bLdBj$"$j$^$;$s!#$b$C$H$b!"$=e(B  
e$B$NJ,8zN($O$h$/$J$$$N$G$9$,!"%M%$%F%#%VMQ$N$HFs=E4IM}$OLLE]$@$C$?e(B  
e$B$N$G!#e(B

e$B$b$7%M%$%F%#%VMQ$HJ,$1$k$J$i!"$3$s$J46$8$K$J$j$^$9!#e(B

require ‘mkmf’

base\_fig = 0  
src = ("(BASE \> 0) && "  
“(BASE \* (BASE+1)) \> BASE && "  
“(BASE \* (BASE+1)) / BASE == (BASE+1)”)  
if CROSS\_COMPILING  
while try\_static\_assert(src, nil, “-DBASE=10#{‘0’\*base\_fig}UL”)  
base\_fig += 1  
end  
else  
src = %{#{COMMON\_HEADERS}  
#include \<stdio.h\>  
/_top_/  
int  
conftest\_value()  
{  
int BASE = 10, base\_fig = 0;  
while (#{src}) {  
BASE \*= 10;  
base\_fig++;  
}  
return base\_fig;  
}  
int main() {printf(”%d\n", conftest\_value()); return 0;}  
}  
if try\_link0(src)  
xpopen("./conftest") do |f|  
base\_fig = Integer(f.gets)  
end  
end  
end  
$defs \<\< “-DBASE=1#{‘0’\*base\_fig}UL” \<\< “-DBASE\_FIG=#{base\_fig}”

create\_makefile(‘bigdecimal’)

---

<div class="post-metadata">

**Author:** ![nobu](https://www.ruby-forum.com/letter_avatar_proxy/v4/letter/n/e9c0ed/32.png) [@nobu](https://www.ruby-forum.com/u/nobu)\
**Post date:** [February 9, 2008, 1:22pm UTC](https://www.ruby-forum.com/t/bigdecimal-to-f-precision-issue/128523/4 "2008-02-09T13:22:44Z")

</div>

e$B\>.NS$G$9!#e(B

----- Original Message -----  
From: “Nobuyoshi N.” [removed\_email\_address@domain.invalid](mailto:removed_email_address@domain.invalid)  
Subject: [ruby-dev:33658] BigDecimal#to\_f precision issue

> [ruby-talk:290296]e$B$NOC$G$9$,!“e(BBigDecimal#to\_fe$B$N@:EY$,IT==J,$J$h$&$G$9!#e(B  
> e$B#F#l#o#a#t$H$NAj8\_JQ49Ey$O$”$^$j5$$K$7$F$$$J$+$C$?$G$9!#e(B  
> e$B$^$5$+=[4D\>.?t$K$J$i$J$$%1!\<%9$Ge(B ==  
> e$BH=Dj$re(BFloate$B$G$9$kEy$H$$$&$3$H$re(B  
> e$B$d$k?M$,$$$k$H$O;W$$$^$;$s$G$7$?!#6C$-!J46?4$7!K$^$7$?!#e(B

> e$B$b;W$$$^$7$?$,!"e(Bstrtod()e$B$r$=$N$^$^;H$&$N$,0lHV4JC1$8$c$J$$$G$7$ge(B  
> e$B$&$+!#e(B  
> e$B3N$+$K!D!#e(B

> e$B$^$?!“e(BDBLE\_FIGe$B$rFbIt$G7W;;$7$F$$$^$9$,!”%l%8%9%?4V$N1i;;$K:GE,2=e(B  
> e$B$5$l$?\>l9g!“e(Bdoublee$B$G$O$J$/$Fe(Blong doublee$B$G7W;;$5$l$F$7$^$&\>l9g$,e(B  
> e$B$”$j$^$9!#e(BDBL\_DIGe$B$r;H$&$Y$-$G$7$g$&!#e(B  
> X86e$BEy$N%l%8%9%?$G$h$jD9$$7e?t$G7W;;$5$l$k$3$H$r4|BT$7$F:#e(B  
> e$B$NMM$K$J$C$F$$$^$9$,!"e(BDBL\_DIGe$B!#1$G$b$h$m$7$$$+$H;W$$$^$9!#e(B

> e$B$b$&0lE@!“e(BBASEe$B$de(BBASE\_FIGe$B$b\<B9T;~$K5a$a$k$h$j$bDj?t$K$7$?$[$&$,$$e(B  
> e$B$$$N$G$O$J$$$+$H;W$$$^$9!#e(B  
> e$B$3$3$,!”$A$g$C$H$“$d$U$d$J$N$G$9$,!”#6#4#B#i#t@0?tEy$N$H$-$OBg\>fIWe(B  
> e$B$G$7$g$&$+!)!!#3#2#B#i#t0J\>e$N@0?t$G$O!"$=$l$J$j$Ne(B BASE  
> e$B$K$7$J$$$He(B  
> e$B8zN($,0-$/$J$j$^$9!#e(B e$BI=8=2DG=$J@0?t$Ke(B BASE\*\*2  
> e$B$,\<#$^$k$H$7$F!"e(B  
> e$B:GBgCM$re(B BASE  
> e$B$K$9$k$h$&$K$J$C$F$$$^$9!J$?$@$7!"e(BBASEe$B$O#1#0$Ne(B  
> e$B$Y$-\>h!K!#e(B

e$B%Q%C%A$H%3%\_%C%H$"$j$,$H$&$4$6$$$^$9!#e(B  
e$B0J\>e$G$9!#e(B