diff --git a/ChangeLog b/ChangeLog index afb5155..b785060 100644 --- a/ChangeLog +++ b/ChangeLog @@ -4,7 +4,7 @@ Next Release 0.15 Deprecated and removed features: -------------------------------- -...none yet... +* array_list_new() has been deprecated in favor of array_list_new2() Other changes -------------- @@ -19,6 +19,12 @@ Other changes less memory usage. Memory used just for json_object structures decreased 27%, so use cases with fewer arrays and/or strings would benefit more. +* Minimize memory usage in array handling in json_tokener by shrinking + arrays to the exact number of elements parsed. On bench/ benchmark: + 9% faster test time, 39%(max RSS)-50%(peak heap) less memory usage. + Add json_object_array_shrink() and array_list_shrink() functions. +* Add json_object_new_array_ext(int) and array_list_new_2(int) to allow + arrays to be allocated with the exact size needed, when known. *** diff --git a/arraylist.c b/arraylist.c index 3e7bfa8..c21b8e1 100644 --- a/arraylist.c +++ b/arraylist.c @@ -37,13 +37,18 @@ #include "arraylist.h" struct array_list *array_list_new(array_list_free_fn *free_fn) +{ + return array_list_new2(free_fn, ARRAY_LIST_DEFAULT_SIZE); +} + +struct array_list *array_list_new2(array_list_free_fn *free_fn, int initial_size) { struct array_list *arr; arr = (struct array_list *)malloc(sizeof(struct array_list)); if (!arr) return NULL; - arr->size = ARRAY_LIST_DEFAULT_SIZE; + arr->size = initial_size; arr->length = 0; arr->free_fn = free_fn; if (!(arr->array = (void **)malloc(arr->size * sizeof(void *)))) @@ -96,6 +101,26 @@ static int array_list_expand_internal(struct array_list *arr, size_t max) return 0; } +int array_list_shrink(struct array_list *arr, size_t empty_slots) +{ + void *t; + size_t new_size; + + new_size = arr->length + empty_slots; + if (new_size == arr->size) + return 0; + if (new_size > arr->size) + return array_list_expand_internal(arr, new_size); + if (new_size == 0) + new_size = 1; + + if (!(t = realloc(arr->array, new_size * sizeof(void *)))) + return -1; + arr->array = (void **)t; + arr->size = new_size; + return 0; +} + //static inline int _array_list_put_idx(struct array_list *arr, size_t idx, void *data) int array_list_put_idx(struct array_list *arr, size_t idx, void *data) { @@ -165,6 +190,8 @@ int array_list_del_idx(struct array_list *arr, size_t idx, size_t count) return -1; for (i = idx; i < stop; ++i) { + // Because put_idx can skip entries, we need to check if + // there's actually anything in each slot we're erasing. if (arr->array[i]) arr->free_fn(arr->array[i]); } diff --git a/arraylist.h b/arraylist.h index 3c4b1b2..2c0fe93 100644 --- a/arraylist.h +++ b/arraylist.h @@ -37,8 +37,27 @@ struct array_list }; typedef struct array_list array_list; +/** + * Allocate an array_list of the default size (32). + * @deprecated Use array_list_new2() instead. + */ extern struct array_list *array_list_new(array_list_free_fn *free_fn); +/** + * Allocate an array_list of the desired size. + * + * If possible, the size should be chosen to closely match + * the actual number of elements expected to be used. + * If the exact size is unknown, there are tradeoffs to be made: + * - too small - the array_list code will need to call realloc() more + * often (which might incur an additional memory copy). + * - too large - will waste memory, but that can be mitigated + * by calling array_list_shrink() once the final size is known. + * + * @see array_list_shrink + */ +extern struct array_list *array_list_new2(array_list_free_fn *free_fn, int initial_size); + extern void array_list_free(struct array_list *al); extern void *array_list_get_idx(struct array_list *al, size_t i); @@ -56,6 +75,14 @@ extern void *array_list_bsearch(const void **key, struct array_list *arr, extern int array_list_del_idx(struct array_list *arr, size_t idx, size_t count); + +/** + * Shrink the array list to just enough to fit the number of elements in it, + * plus empty_slots. + */ +extern int array_list_shrink(struct array_list *arr, size_t empty_slots); + + #ifdef __cplusplus } #endif diff --git a/json_object.c b/json_object.c index be84cb0..89439ab 100644 --- a/json_object.c +++ b/json_object.c @@ -1431,11 +1431,15 @@ static void json_object_array_delete(struct json_object *jso) } struct json_object *json_object_new_array(void) +{ + return json_object_new_array_ext(ARRAY_LIST_DEFAULT_SIZE); +} +struct json_object *json_object_new_array_ext(int initial_size) { struct json_object_array *jso = JSON_OBJECT_NEW(array); if (!jso) return NULL; - jso->c_array = array_list_new(&json_object_array_entry_free); + jso->c_array = array_list_new2(&json_object_array_entry_free, initial_size); if (jso->c_array == NULL) { free(jso); @@ -1523,6 +1527,13 @@ static int json_array_equal(struct json_object *jso1, struct json_object *jso2) return 1; } +int json_object_array_shrink(struct json_object *jso, int empty_slots) +{ + if (empty_slots < 0) + json_abort("json_object_array_shrink called with negative empty_slots"); + return array_list_shrink(JC_ARRAY(jso)->c_array, empty_slots); +} + struct json_object *json_object_new_null(void) { return NULL; diff --git a/json_object.h b/json_object.h index 7c0d1f2..5f2f64c 100644 --- a/json_object.h +++ b/json_object.h @@ -500,10 +500,16 @@ JSON_EXPORT void json_object_object_del(struct json_object *obj, const char *key /* Array type methods */ /** Create a new empty json_object of type json_type_array + * If you know the array size you'll need ahead of time, use + * json_object_new_array_ext() instead. + * @see json_object_new_array_ext() + * @see json_object_array_shrink() * @returns a json_object of type json_type_array */ JSON_EXPORT struct json_object *json_object_new_array(void); +JSON_EXPORT struct json_object *json_object_new_array_ext(int initial_size); + /** Get the arraylist of a json_object of type json_type_array * @param obj the json_object instance * @returns an arraylist @@ -595,6 +601,15 @@ JSON_EXPORT struct json_object *json_object_array_get_idx(const struct json_obje */ JSON_EXPORT int json_object_array_del_idx(struct json_object *obj, size_t idx, size_t count); +/** + * Shrink the internal memory allocation of the array to just + * enough to fit the number of elements in it, plus empty_slots. + * + * @param jso the json_object instance, must be json_type_array + * @param empty_slots the number of empty slots to leave allocated + */ +JSON_EXPORT int json_object_array_shrink(struct json_object *jso, int empty_slots); + /* json_bool type methods */ /** Create a new empty json_object of type json_type_boolean diff --git a/json_tokener.c b/json_tokener.c index 0373d6f..9aa86a9 100644 --- a/json_tokener.c +++ b/json_tokener.c @@ -964,6 +964,9 @@ struct json_object *json_tokener_parse_ex(struct json_tokener *tok, const char * case json_tokener_state_array: if (c == ']') { + // Minimize memory usage; assume parsed objs are unlikely to be changed + json_object_array_shrink(current, 0); + if (state == json_tokener_state_array_after_sep && (tok->flags & JSON_TOKENER_STRICT)) { @@ -997,6 +1000,9 @@ struct json_object *json_tokener_parse_ex(struct json_tokener *tok, const char * case json_tokener_state_array_sep: if (c == ']') { + // Minimize memory usage; assume parsed objs are unlikely to be changed + json_object_array_shrink(current, 0); + saved_state = json_tokener_state_finish; state = json_tokener_state_eatws; }