Fix float precision (#649)

The issue is that numbers like
2.01 or 3.01 can not be precisely represented with binary floating point
numbers.

This replaces all occurrences of 'std::numeric_limits<T>::digits10 + 1' with
'std::numeric_limits<T>::max_digits10'.

Background:
Using 'std::numeric_limits<T>::digits10 + 1' is not precise enough.
Converting a 'float' into a 'string' and back to a 'float' will not always
produce the original 'float' value. To guarantee that the 'string'
representation has sufficient precision the value
'std::numeric_limits<T>::max_digits10' has to be used.
This commit is contained in:
Simon Gene Gottlieb
2018-12-21 15:05:19 +01:00
committed by Jesse Beder
parent b659858b19
commit abf941b20d
4 changed files with 55 additions and 49 deletions

View File

@@ -93,7 +93,7 @@ struct convert<_Null> {
struct convert<type> { \ struct convert<type> { \
static Node encode(const type& rhs) { \ static Node encode(const type& rhs) { \
std::stringstream stream; \ std::stringstream stream; \
stream.precision(std::numeric_limits<type>::digits10 + 1); \ stream.precision(std::numeric_limits<type>::max_digits10); \
stream << rhs; \ stream << rhs; \
return Node(stream.str()); \ return Node(stream.str()); \
} \ } \

View File

@@ -24,8 +24,8 @@ EmitterState::EmitterState()
m_seqFmt.set(Block); m_seqFmt.set(Block);
m_mapFmt.set(Block); m_mapFmt.set(Block);
m_mapKeyFmt.set(Auto); m_mapKeyFmt.set(Auto);
m_floatPrecision.set(std::numeric_limits<float>::digits10 + 1); m_floatPrecision.set(std::numeric_limits<float>::max_digits10);
m_doublePrecision.set(std::numeric_limits<double>::digits10 + 1); m_doublePrecision.set(std::numeric_limits<double>::max_digits10);
} }
EmitterState::~EmitterState() {} EmitterState::~EmitterState() {}
@@ -349,7 +349,7 @@ bool EmitterState::SetMapKeyFormat(EMITTER_MANIP value, FmtScope::value scope) {
} }
bool EmitterState::SetFloatPrecision(std::size_t value, FmtScope::value scope) { bool EmitterState::SetFloatPrecision(std::size_t value, FmtScope::value scope) {
if (value > std::numeric_limits<float>::digits10 + 1) if (value > std::numeric_limits<float>::max_digits10)
return false; return false;
_Set(m_floatPrecision, value, scope); _Set(m_floatPrecision, value, scope);
return true; return true;
@@ -357,7 +357,7 @@ bool EmitterState::SetFloatPrecision(std::size_t value, FmtScope::value scope) {
bool EmitterState::SetDoublePrecision(std::size_t value, bool EmitterState::SetDoublePrecision(std::size_t value,
FmtScope::value scope) { FmtScope::value scope) {
if (value > std::numeric_limits<double>::digits10 + 1) if (value > std::numeric_limits<double>::max_digits10)
return false; return false;
_Set(m_doublePrecision, value, scope); _Set(m_doublePrecision, value, scope);
return true; return true;

View File

@@ -901,18 +901,18 @@ TEST_F(EmitterTest, SingleChar) {
TEST_F(EmitterTest, DefaultPrecision) { TEST_F(EmitterTest, DefaultPrecision) {
out << BeginSeq; out << BeginSeq;
out << 1.234f; out << 1.3125f;
out << 3.14159265358979; out << 1.23455810546875;
out << EndSeq; out << EndSeq;
ExpectEmit("- 1.234\n- 3.14159265358979"); ExpectEmit("- 1.3125\n- 1.23455810546875");
} }
TEST_F(EmitterTest, SetPrecision) { TEST_F(EmitterTest, SetPrecision) {
out << BeginSeq; out << BeginSeq;
out << FloatPrecision(3) << 1.234f; out << FloatPrecision(3) << 1.3125f;
out << DoublePrecision(6) << 3.14159265358979; out << DoublePrecision(6) << 1.23455810546875;
out << EndSeq; out << EndSeq;
ExpectEmit("- 1.23\n- 3.14159"); ExpectEmit("- 1.31\n- 1.23456");
} }
TEST_F(EmitterTest, DashInBlockContext) { TEST_F(EmitterTest, DashInBlockContext) {

View File

@@ -391,7 +391,13 @@ TEST(NodeTest, AutoBoolConversion) {
EXPECT_TRUE(!!node["foo"]); EXPECT_TRUE(!!node["foo"]);
} }
TEST(NodeTest, FloatingPrecision) { TEST(NodeTest, FloatingPrecisionFloat) {
const float x = 0.123456789;
Node node = Node(x);
EXPECT_EQ(x, node.as<float>());
}
TEST(NodeTest, FloatingPrecisionDouble) {
const double x = 0.123456789; const double x = 0.123456789;
Node node = Node(x); Node node = Node(x);
EXPECT_EQ(x, node.as<double>()); EXPECT_EQ(x, node.as<double>());
@@ -452,108 +458,108 @@ class NodeEmitterTest : public ::testing::Test {
TEST_F(NodeEmitterTest, SimpleFlowSeqNode) { TEST_F(NodeEmitterTest, SimpleFlowSeqNode) {
Node node; Node node;
node.SetStyle(EmitterStyle::Flow); node.SetStyle(EmitterStyle::Flow);
node.push_back(1.01); node.push_back(1.5);
node.push_back(2.01); node.push_back(2.25);
node.push_back(3.01); node.push_back(3.125);
ExpectOutput("[1.01, 2.01, 3.01]", node); ExpectOutput("[1.5, 2.25, 3.125]", node);
} }
TEST_F(NodeEmitterTest, NestFlowSeqNode) { TEST_F(NodeEmitterTest, NestFlowSeqNode) {
Node node, cell0, cell1; Node node, cell0, cell1;
cell0.push_back(1.01); cell0.push_back(1.5);
cell0.push_back(2.01); cell0.push_back(2.25);
cell0.push_back(3.01); cell0.push_back(3.125);
cell1.push_back(4.01); cell1.push_back(4.5);
cell1.push_back(5.01); cell1.push_back(5.25);
cell1.push_back(6.01); cell1.push_back(6.125);
node.SetStyle(EmitterStyle::Flow); node.SetStyle(EmitterStyle::Flow);
node.push_back(cell0); node.push_back(cell0);
node.push_back(cell1); node.push_back(cell1);
ExpectOutput("[[1.01, 2.01, 3.01], [4.01, 5.01, 6.01]]", node); ExpectOutput("[[1.5, 2.25, 3.125], [4.5, 5.25, 6.125]]", node);
} }
TEST_F(NodeEmitterTest, MixBlockFlowSeqNode) { TEST_F(NodeEmitterTest, MixBlockFlowSeqNode) {
Node node, cell0, cell1; Node node, cell0, cell1;
cell0.SetStyle(EmitterStyle::Flow); cell0.SetStyle(EmitterStyle::Flow);
cell0.push_back(1.01); cell0.push_back(1.5);
cell0.push_back(2.01); cell0.push_back(2.25);
cell0.push_back(3.01); cell0.push_back(3.125);
cell1.push_back(4.01); cell1.push_back(4.5);
cell1.push_back(5.01); cell1.push_back(5.25);
cell1.push_back(6.01); cell1.push_back(6.125);
node.SetStyle(EmitterStyle::Block); node.SetStyle(EmitterStyle::Block);
node.push_back(cell0); node.push_back(cell0);
node.push_back(cell1); node.push_back(cell1);
ExpectOutput("- [1.01, 2.01, 3.01]\n-\n - 4.01\n - 5.01\n - 6.01", node); ExpectOutput("- [1.5, 2.25, 3.125]\n-\n - 4.5\n - 5.25\n - 6.125", node);
} }
TEST_F(NodeEmitterTest, NestBlockFlowMapListNode) { TEST_F(NodeEmitterTest, NestBlockFlowMapListNode) {
Node node, mapNode, blockNode; Node node, mapNode, blockNode;
node.push_back(1.01); node.push_back(1.5);
node.push_back(2.01); node.push_back(2.25);
node.push_back(3.01); node.push_back(3.125);
mapNode.SetStyle(EmitterStyle::Flow); mapNode.SetStyle(EmitterStyle::Flow);
mapNode["position"] = node; mapNode["position"] = node;
blockNode.push_back(1.01); blockNode.push_back(1.0625);
blockNode.push_back(mapNode); blockNode.push_back(mapNode);
ExpectOutput("- 1.01\n- {position: [1.01, 2.01, 3.01]}", blockNode); ExpectOutput("- 1.0625\n- {position: [1.5, 2.25, 3.125]}", blockNode);
} }
TEST_F(NodeEmitterTest, NestBlockMixMapListNode) { TEST_F(NodeEmitterTest, NestBlockMixMapListNode) {
Node node, mapNode, blockNode; Node node, mapNode, blockNode;
node.push_back(1.01); node.push_back(1.5);
node.push_back(2.01); node.push_back(2.25);
node.push_back(3.01); node.push_back(3.125);
mapNode.SetStyle(EmitterStyle::Flow); mapNode.SetStyle(EmitterStyle::Flow);
mapNode["position"] = node; mapNode["position"] = node;
blockNode["scalar"] = 1.01; blockNode["scalar"] = 1.0625;
blockNode["object"] = mapNode; blockNode["object"] = mapNode;
ExpectAnyOutput(blockNode, ExpectAnyOutput(blockNode,
"scalar: 1.01\nobject: {position: [1.01, 2.01, 3.01]}", "scalar: 1.0625\nobject: {position: [1.5, 2.25, 3.125]}",
"object: {position: [1.01, 2.01, 3.01]}\nscalar: 1.01"); "object: {position: [1.5, 2.25, 3.125]}\nscalar: 1.5");
} }
TEST_F(NodeEmitterTest, NestBlockMapListNode) { TEST_F(NodeEmitterTest, NestBlockMapListNode) {
Node node, mapNode; Node node, mapNode;
node.push_back(1.01); node.push_back(1.5);
node.push_back(2.01); node.push_back(2.25);
node.push_back(3.01); node.push_back(3.125);
mapNode.SetStyle(EmitterStyle::Block); mapNode.SetStyle(EmitterStyle::Block);
mapNode["position"] = node; mapNode["position"] = node;
ExpectOutput("position:\n - 1.01\n - 2.01\n - 3.01", mapNode); ExpectOutput("position:\n - 1.5\n - 2.25\n - 3.125", mapNode);
} }
TEST_F(NodeEmitterTest, NestFlowMapListNode) { TEST_F(NodeEmitterTest, NestFlowMapListNode) {
Node node, mapNode; Node node, mapNode;
node.push_back(1.01); node.push_back(1.5);
node.push_back(2.01); node.push_back(2.25);
node.push_back(3.01); node.push_back(3.125);
mapNode.SetStyle(EmitterStyle::Flow); mapNode.SetStyle(EmitterStyle::Flow);
mapNode["position"] = node; mapNode["position"] = node;
ExpectOutput("{position: [1.01, 2.01, 3.01]}", mapNode); ExpectOutput("{position: [1.5, 2.25, 3.125]}", mapNode);
} }
} }
} }