From 16a4a32e294e80ca8c89ec83e62baa0c59947ac9 Mon Sep 17 00:00:00 2001 From: Remi Collet Date: Tue, 27 Nov 2012 11:06:49 +0100 Subject: [PATCH 1/5] float parsing must be locale independent --- configure.in | 4 ++-- json_tokener.c | 2 +- json_util.c | 25 +++++++++++++++++++++++++ json_util.h | 1 + 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/configure.in b/configure.in index 387b422..ea3ed67 100644 --- a/configure.in +++ b/configure.in @@ -23,7 +23,7 @@ AM_CONDITIONAL(ENABLE_OLDNAME_COMPAT, [test "x${enable_oldname_compat}" != "xno" AM_CONFIG_HEADER(config.h) AM_CONFIG_HEADER(json_config.h) AC_HEADER_STDC -AC_CHECK_HEADERS(fcntl.h limits.h strings.h syslog.h unistd.h [sys/cdefs.h] [sys/param.h] stdarg.h) +AC_CHECK_HEADERS(fcntl.h limits.h strings.h syslog.h unistd.h [sys/cdefs.h] [sys/param.h] stdarg.h locale.h) AC_CHECK_HEADER(inttypes.h,[AC_DEFINE([JSON_C_HAVE_INTTYPES_H],[1],[Public define for json_inttypes.h])]) # Checks for typedefs, structures, and compiler characteristics. @@ -35,7 +35,7 @@ AC_FUNC_VPRINTF AC_FUNC_MEMCMP AC_FUNC_MALLOC AC_FUNC_REALLOC -AC_CHECK_FUNCS(strcasecmp strdup strndup strerror snprintf vsnprintf vasprintf open vsyslog strncasecmp) +AC_CHECK_FUNCS(strcasecmp strdup strndup strerror snprintf vsnprintf vasprintf open vsyslog strncasecmp setlocale) #check if .section.gnu.warning accepts long strings (for __warn_references) AC_LANG_PUSH([C]) diff --git a/json_tokener.c b/json_tokener.c index f5fa8d6..85c530b 100644 --- a/json_tokener.c +++ b/json_tokener.c @@ -585,7 +585,7 @@ struct json_object* json_tokener_parse_ex(struct json_tokener *tok, double numd; if (!tok->is_double && json_parse_int64(tok->pb->buf, &num64) == 0) { current = json_object_new_int64(num64); - } else if(tok->is_double && sscanf(tok->pb->buf, "%lf", &numd) == 1) { + } else if(tok->is_double && json_parse_double(tok->pb->buf, &numd) == 0) { current = json_object_new_double(numd); } else { tok->err = json_tokener_error_parse_number; diff --git a/json_util.c b/json_util.c index 79ae5c7..c144059 100644 --- a/json_util.c +++ b/json_util.c @@ -36,6 +36,10 @@ # include #endif /* HAVE_UNISTD_H */ +#ifdef HAVE_LOCALE_H +#include +#endif /* HAVE_LOCALE_H */ + #ifdef WIN32 # define WIN32_LEAN_AND_MEAN # include @@ -142,6 +146,27 @@ int json_object_to_file(char *filename, struct json_object *obj) return json_object_to_file_ext(filename, obj, JSON_C_TO_STRING_PLAIN); } +int json_parse_double(const char *buf, double *retval) +{ + int ret; +#ifdef HAVE_SETLOCALE + char *old=NULL, *tmp; + + tmp = setlocale(LC_NUMERIC, NULL); + if (tmp) old = strdup(tmp); + setlocale(LC_NUMERIC, "C"); +#endif + + ret = sscanf(buf, "%lf", retval); + +#ifdef HAVE_SETLOCALE + setlocale(LC_NUMERIC, old); + if (old) free(old); +#endif + + return (ret==1 ? 0 : 1); +} + int json_parse_int64(const char *buf, int64_t *retval) { int64_t num64; diff --git a/json_util.h b/json_util.h index 277c3a7..b9a69c8 100644 --- a/json_util.h +++ b/json_util.h @@ -25,6 +25,7 @@ extern struct json_object* json_object_from_file(const char *filename); extern int json_object_to_file(char *filename, struct json_object *obj); extern int json_object_to_file_ext(char *filename, struct json_object *obj, int flags); extern int json_parse_int64(const char *buf, int64_t *retval); +extern int json_parse_double(const char *buf, double *retval); /** From a01b659ace168d85a3e9e47848eaaba2bea31078 Mon Sep 17 00:00:00 2001 From: Remi Collet Date: Thu, 13 Dec 2012 09:47:33 +0100 Subject: [PATCH 2/5] move locale change to be global for perf --- json_tokener.c | 16 ++++++++++++++++ json_util.c | 22 +--------------------- tests/Makefile.am | 3 +++ tests/test_locale.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 tests/test_locale.c diff --git a/json_tokener.c b/json_tokener.c index 85c530b..63bb41b 100644 --- a/json_tokener.c +++ b/json_tokener.c @@ -31,6 +31,10 @@ #include "json_tokener.h" #include "json_util.h" +#ifdef HAVE_LOCALE_H +#include +#endif /* HAVE_LOCALE_H */ + #if !HAVE_STRDUP && defined(_MSC_VER) /* MSC has the version as _strdup */ # define strdup _strdup @@ -227,6 +231,13 @@ struct json_object* json_tokener_parse_ex(struct json_tokener *tok, { struct json_object *obj = NULL; char c = '\1'; +#ifdef HAVE_SETLOCALE + char *oldlocale=NULL, *tmplocale; + + tmplocale = setlocale(LC_NUMERIC, NULL); + if (tmplocale) oldlocale = strdup(tmplocale); + setlocale(LC_NUMERIC, "C"); +#endif tok->char_offset = 0; tok->err = json_tokener_success; @@ -724,6 +735,11 @@ struct json_object* json_tokener_parse_ex(struct json_tokener *tok, tok->err = json_tokener_error_parse_eof; } +#ifdef HAVE_SETLOCALE + setlocale(LC_NUMERIC, oldlocale); + if (oldlocale) free(oldlocale); +#endif + if (tok->err == json_tokener_success) { json_object *ret = json_object_get(current); diff --git a/json_util.c b/json_util.c index c144059..0a59811 100644 --- a/json_util.c +++ b/json_util.c @@ -36,10 +36,6 @@ # include #endif /* HAVE_UNISTD_H */ -#ifdef HAVE_LOCALE_H -#include -#endif /* HAVE_LOCALE_H */ - #ifdef WIN32 # define WIN32_LEAN_AND_MEAN # include @@ -148,23 +144,7 @@ int json_object_to_file(char *filename, struct json_object *obj) int json_parse_double(const char *buf, double *retval) { - int ret; -#ifdef HAVE_SETLOCALE - char *old=NULL, *tmp; - - tmp = setlocale(LC_NUMERIC, NULL); - if (tmp) old = strdup(tmp); - setlocale(LC_NUMERIC, "C"); -#endif - - ret = sscanf(buf, "%lf", retval); - -#ifdef HAVE_SETLOCALE - setlocale(LC_NUMERIC, old); - if (old) free(old); -#endif - - return (ret==1 ? 0 : 1); + return (sscanf(buf, "%lf", retval)==1 ? 0 : 1); } int json_parse_int64(const char *buf, int64_t *retval) diff --git a/tests/Makefile.am b/tests/Makefile.am index 8057acd..90222ba 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -11,6 +11,7 @@ check_PROGRAMS += test_parse_int64 check_PROGRAMS += test_null check_PROGRAMS += test_cast check_PROGRAMS += test_parse +check_PROGRAMS += test_locale test1_LDADD = $(LIBJSON_LA) @@ -36,6 +37,8 @@ test_cast_LDADD = $(LIBJSON_LA) test_parse_LDADD = $(LIBJSON_LA) +test_locale_LDADD = $(LIBJSON_LA) + TESTS = test1.test test2.test test4.test testReplaceExisting.test parse_int64.test test_null.test test_cast.test test_parse.test TESTS+= test_printbuf.test diff --git a/tests/test_locale.c b/tests/test_locale.c new file mode 100644 index 0000000..6926a5d --- /dev/null +++ b/tests/test_locale.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +#include "config.h" +#include "json.h" +#include "json_tokener.h" + +#ifdef HAVE_LOCALE_H +#include +#endif /* HAVE_LOCALE_H */ + +int main(int argc, char **argv) +{ + json_object *new_obj; +#ifdef HAVE_SETLOCALE + setlocale(LC_NUMERIC, "de_DE"); +#else + printf("No locale\n"); +#endif + + MC_SET_DEBUG(1); + + new_obj = json_tokener_parse("[1.2,3.4,123456.78,5.0]"); + printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj)); + json_object_put(new_obj); +} + From 4014fe86d96d850271a1bd724ccf5a83d27dceb8 Mon Sep 17 00:00:00 2001 From: Remi Collet Date: Thu, 13 Dec 2012 11:16:03 +0100 Subject: [PATCH 3/5] Simple fix to double encode --- json_object.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/json_object.c b/json_object.c index 5b60a06..eeba91b 100644 --- a/json_object.c +++ b/json_object.c @@ -552,7 +552,16 @@ static int json_object_double_to_json_string(struct json_object* jso, int level, int flags) { - return sprintbuf(pb, "%f", jso->o.c_double); + char buf[128], *p; + int size; + + size = snprintf(buf, 128, "%f", jso->o.c_double); + p = strchr(buf, ','); + if (p) { + *p = '.'; + } + printbuf_memappend(pb, buf, size); + return size; } struct json_object* json_object_new_double(double d) From 8c847968c7de17be6cf3878a4e6adca8ab8d8f14 Mon Sep 17 00:00:00 2001 From: Remi Collet Date: Thu, 13 Dec 2012 11:22:31 +0100 Subject: [PATCH 4/5] Save space, drop unuseful trailing zeroes --- json_object.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/json_object.c b/json_object.c index eeba91b..b078017 100644 --- a/json_object.c +++ b/json_object.c @@ -552,13 +552,25 @@ static int json_object_double_to_json_string(struct json_object* jso, int level, int flags) { - char buf[128], *p; + char buf[128], *p, *q; int size; size = snprintf(buf, 128, "%f", jso->o.c_double); p = strchr(buf, ','); if (p) { *p = '.'; + } else { + p = strchr(buf, '.'); + } + if (p) { + /* last useful digit, always keep 1 zero */ + p++; + for (q=p ; *q ; q++) { + if (*q!='0') p=q; + } + /* drop trailing zeroes */ + *(++p) = 0; + size = p-buf; } printbuf_memappend(pb, buf, size); return size; From 32d149c8f60a5bdad4f678013a83d052ffd8ad43 Mon Sep 17 00:00:00 2001 From: Remi Collet Date: Thu, 13 Dec 2012 11:46:04 +0100 Subject: [PATCH 5/5] probably worth an option for this --- json_object.c | 2 +- json_object.h | 4 ++++ tests/test_locale.c | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/json_object.c b/json_object.c index b078017..9161b00 100644 --- a/json_object.c +++ b/json_object.c @@ -562,7 +562,7 @@ static int json_object_double_to_json_string(struct json_object* jso, } else { p = strchr(buf, '.'); } - if (p) { + if (p && (flags & JSON_C_TO_STRING_NOZERO)) { /* last useful digit, always keep 1 zero */ p++; for (q=p ; *q ; q++) { diff --git a/json_object.h b/json_object.h index 2621112..1f70461 100644 --- a/json_object.h +++ b/json_object.h @@ -42,6 +42,10 @@ extern "C" { * for an example of the format. */ #define JSON_C_TO_STRING_PRETTY (1<<1) +/** + * A flag to drop trailing zero for float values + */ +#define JSON_C_TO_STRING_NOZERO (1<<2) #undef FALSE #define FALSE ((json_bool)0) diff --git a/tests/test_locale.c b/tests/test_locale.c index 6926a5d..da070cf 100644 --- a/tests/test_locale.c +++ b/tests/test_locale.c @@ -23,8 +23,9 @@ int main(int argc, char **argv) MC_SET_DEBUG(1); - new_obj = json_tokener_parse("[1.2,3.4,123456.78,5.0]"); + new_obj = json_tokener_parse("[1.2,3.4,123456.78,5.0,2.3e10]"); printf("new_obj.to_string()=%s\n", json_object_to_json_string(new_obj)); + printf("new_obj.to_string()=%s\n", json_object_to_json_string_ext(new_obj,JSON_C_TO_STRING_NOZERO)); json_object_put(new_obj); }