Forum Discussion

will_fetherolf's avatar
will_fetherolf
Contributor
10 years ago

Odd issue with aqString.Format and floats

Consider the following code:   function formattest() { var a = 13.326; var c = 13.325; var b = .325;   Log.Message(aqString.Format("%#1.1f",a)); Log.Message(aqString.Format("%#0.2f",a)); Lo...
  • joseph_michaud's avatar
    joseph_michaud
    10 years ago

    All is not as it seems.  There's a useful little numeric converter that helps you see how a floating point number is stored

     

    IEEE 754 Converter

     

    If you expand the displayed digits for the value used to represent your number, you can see why the seeminly odd rounding is occurring:

     

    function formattest()
{
  var vals = [  "0.305",  "0.315",  "0.325",  "0.335",  "0.345",
                    "0.355",  "0.365",  "0.375",  "0.385",  "0.395",
               "13.305", "13.315", "13.325", "13.335", "13.345",
                   "13.355", "13.365", "13.375", "13.385", "13.395" ];
  var ta;
  for (var i = 0; i < vals.length; i++) {
    ta = vals[i]; Log.Message(aqString.Format("%s = %.19f = %0.2f", ta, ta, ta));
  }
}

    produces the following output:

     

    0.305 = 0.3049999999999999900 = 0.30
0.315 = 0.3150000000000000000 = 0.32
0.325 = 0.3250000000000000100 = 0.33
0.335 = 0.3350000000000000200 = 0.34
0.345 = 0.3449999999999999700 = 0.34
0.355 = 0.3549999999999999800 = 0.35
0.365 = 0.3649999999999999900 = 0.36
0.375 = 0.3750000000000000000 = 0.38
0.385 = 0.3850000000000000100 = 0.39
0.395 = 0.3950000000000000200 = 0.40
13.305 = 13.3050000000000000000 = 13.31
13.315 = 13.3150000000000000000 = 13.32
13.325 = 13.3249999999999990000 = 13.32
13.335 = 13.3350000000000010000 = 13.34
13.345 = 13.3450000000000010000 = 13.35
13.355 = 13.3550000000000000000 = 13.36
13.365 = 13.3650000000000000000 = 13.37
13.375 = 13.3750000000000000000 = 13.38
13.385 = 13.3850000000000000000 = 13.39
13.395 = 13.3950000000000000000 = 13.40
joseph_michaud's avatar
joseph_michaud
Moderator

All is not as it seems.  There's a useful little numeric converter that helps you see how a floating point number is stored

 

IEEE 754 Converter

 

If you expand the displayed digits for the value used to represent your number, you can see why the seeminly odd rounding is occurring:

 

function formattest()
{
  var vals = [  "0.305",  "0.315",  "0.325",  "0.335",  "0.345",
                "0.355",  "0.365",  "0.375",  "0.385",  "0.395",
               "13.305", "13.315", "13.325", "13.335", "13.345",
               "13.355", "13.365", "13.375", "13.385", "13.395" ];
  var ta;
  for (var i = 0; i < vals.length; i++) {
    ta = vals[i]; Log.Message(aqString.Format("%s = %.19f = %0.2f", ta, ta, ta));
  }
}

produces the following output:

 

0.305 = 0.3049999999999999900 = 0.30
0.315 = 0.3150000000000000000 = 0.32
0.325 = 0.3250000000000000100 = 0.33
0.335 = 0.3350000000000000200 = 0.34
0.345 = 0.3449999999999999700 = 0.34
0.355 = 0.3549999999999999800 = 0.35
0.365 = 0.3649999999999999900 = 0.36
0.375 = 0.3750000000000000000 = 0.38
0.385 = 0.3850000000000000100 = 0.39
0.395 = 0.3950000000000000200 = 0.40
13.305 = 13.3050000000000000000 = 13.31
13.315 = 13.3150000000000000000 = 13.32
13.325 = 13.3249999999999990000 = 13.32
13.335 = 13.3350000000000010000 = 13.34
13.345 = 13.3450000000000010000 = 13.35
13.355 = 13.3550000000000000000 = 13.36
13.365 = 13.3650000000000000000 = 13.37
13.375 = 13.3750000000000000000 = 13.38
13.385 = 13.3850000000000000000 = 13.39
13.395 = 13.3950000000000000000 = 13.40
will_fetherolf's avatar
will_fetherolf
Contributor
10 years ago

Well, that's infuriating.  I did get around it by running .toFixed against the number to have it properly rounded.

 

function testformatting()
{
  var nums = [13.315,13.316,13.317,13.318,13.319,13.320,13.321,13.322,13.323,13.324,13.325,13.326,13.327];
  var i;
  var ta;
  
  for (i=0; i<nums.length; i++)
  {
    ta = nums[i];  
    Log.Message(aqString.Format("%#0.19f = %#0.2f = %#0.2f", ta, ta, ta.toFixed(2)));
    
  }
}

Produced the following output:

 

13.3150000000000000000 = 13.32 = 13.32	
13.3160000000000010000 = 13.32 = 13.32	
13.3170000000000000000 = 13.32 = 13.32	
13.3180000000000000000 = 13.32 = 13.32	
13.3190000000000010000 = 13.32 = 13.32	
13.3200000000000000000 = 13.32 = 13.32	
13.3210000000000000000 = 13.32 = 13.32	
13.3219999999999990000 = 13.32 = 13.32	
13.3230000000000000000 = 13.32 = 13.32	
13.3240000000000000000 = 13.32 = 13.32	
13.3249999999999990000 = 13.32 = 13.33	
13.3260000000000010000 = 13.33 = 13.33	
13.3270000000000000000 = 13.33 = 13.33