diff --git a/contrib/libucl/ChangeLog.md b/contrib/libucl/ChangeLog.md index f714f40e75e4..e432f84e91d4 100644 --- a/contrib/libucl/ChangeLog.md +++ b/contrib/libucl/ChangeLog.md @@ -37,3 +37,31 @@ - Fixed a bug with macroes that come after an empty object - Fixed a bug in include processing when an incorrect variable has been destroyed (use-after-free) + +### Libucl 0.8.0 + +- Allow to save comments and macros when parsing UCL documents +- C++ API +- Python bindings (by Eitan Adler) +- Add msgpack support for parser and emitter +- Add Canonical S-expressions parser for libucl +- CLI interface for parsing and validation (by Maxim Ignatenko) +- Implement include with priority +- Add 'nested' functionality to .include macro (by Allan Jude) +- Allow searching an array of paths for includes (by Allan Jude) +- Add new .load macro (by Allan Jude) +- Implement .inherit macro (#100) +- Add merge strategies +- Add schema validation to lua API +- Add support for external references to schema validation +- Add coveralls integration to libucl +- Implement tests for 80% of libucl code lines +- Fix tonns of minor and major bugs +- Improve documentation +- Rework function names to the common conventions (old names are preserved for backwards compatibility) +- Add Coverity scan integration +- Add fuzz tests + +**Incompatible changes**: + +- `ucl_object_emit_full` now accepts additional argument `comments` that could be used to emit comments with UCL output \ No newline at end of file diff --git a/contrib/libucl/Makefile.am b/contrib/libucl/Makefile.am index 957e3a2b0f3c..5b51bcc3b468 100644 --- a/contrib/libucl/Makefile.am +++ b/contrib/libucl/Makefile.am @@ -8,4 +8,74 @@ if LUA_SUB LUA_SUBDIR = lua endif -SUBDIRS = src tests utils doc $(LUA_SUBDIR) \ No newline at end of file +COVERAGE_INFO_FILE = $(top_builddir)/coverage.info +COVERAGE_REPORT_DIR = $(top_builddir)/coverage + +.PHONY = coverage-requirement-check clean-coverage-report + +coverage-requirement-check: + @if test ! -e $(GCOV); then \ + echo "Cannot find $(GCOV). Please install gcov."; \ + exit 1; \ + fi + +coverage: coverage-requirement-check clean-coverage coverage-build coverage-check coverage-report + @echo "Please execute 'make clean' before 'make' or 'make check' to remove instrumented object files(compiled with -O0 etc.). Note that 'make clean' also remove coverage data." + +coverage-build: coverage-requirement-check + @if test `find $(top_builddir) -name "*.gcno" | wc -l` -eq 0; then \ + echo "Start to remove old non-instrumented object files..."; \ + $(MAKE) $(AM_MAKEFLAGS) clean; \ + echo "Successfully removed old non-instrumented object files."; \ + fi + @echo "Start to build libraries with coverage options..." + $(MAKE) $(AM_MAKEFLAGS) \ + CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS) $(COVERAGE_OPTFLAGS)" \ + CXXFLAGS="$(CXXFLAGS) $(COVERAGE_CXXFLAGS) $(COVERAGE_OPTFLAGS)" \ + LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" \ + LIBS="$(LIBS) $(COVERAGE_LIBS)" + @echo "Successfully built libraries with coverage options." + +coverage-check: coverage-requirement-check + @echo "Start to run tests with instrumented libraries..." + $(MAKE) $(AM_MAKEFLAGS) check \ + CFLAGS="$(CFLAGS) $(COVERAGE_CFLAGS) $(COVERAGE_OPTFLAGS)" \ + CXXFLAGS="$(CXXFLAGS) $(COVERAGE_CXXFLAGS) $(COVERAGE_OPTFLAGS)" \ + LDFLAGS="$(LDFLAGS) $(COVERAGE_LDFLAGS)" \ + LIBS="$(LIBS) $(COVERAGE_LIBS)" + @echo "Successfully run tests with instrumented libraries." + +coverage-lcov: coverage-check coverage-requirement-check + $(LCOV) --capture \ + --directory "$(top_builddir)/" \ + --output-file $(COVERAGE_INFO_FILE) \ + --gcov-tool $(GCOV) \ + --compat-libtool --checksum + $(LCOV) --extract $(COVERAGE_INFO_FILE) `pwd`/src/ucl_\* \ + --output-file $(COVERAGE_INFO_FILE) + +coverage-report: coverage-lcov + @echo "Start to create coverage reports..." + $(GENHTML) --prefix "$(top_srcdir)" \ + --output-directory $(COVERAGE_REPORT_DIR) \ + --title $(PACKAGE_NAME) \ + --legend --show-details \ + $(GENHTML_OPTIONS) \ + $(COVERAGE_INFO_FILE) + @echo "Successfully created coverage reports into $(COVERAGE_REPORT_DIR) directory." + +clean-coverage-report: + -rm -rf $(COVERAGE_INFO_FILE) + -rm -rf $(COVERAGE_REPORT_DIR) + +clean-coverage: clean-coverage-report + -$(LCOV) --gcov-tool $(GCOV) --zerocounters --directory $(top_builddir) + @if xargs --version 2>/dev/null; then \ + find $(top_builddir) -name "*.gcno" | xargs --no-run-if-empty rm; \ + else \ + find $(top_builddir) -name "*.gcno" | xargs rm; \ + fi + +clean-local: clean-coverage + +SUBDIRS = src tests utils doc $(LUA_SUBDIR) diff --git a/contrib/libucl/README.md b/contrib/libucl/README.md index a742db1d2146..0d76ca4a5e24 100644 --- a/contrib/libucl/README.md +++ b/contrib/libucl/README.md @@ -1,6 +1,6 @@ # LIBUCL -[![Build Status](https://travis-ci.org/vstakhov/libucl.svg?branch=master)](https://travis-ci.org/vstakhov/libucl)[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138) +[![Build Status](https://travis-ci.org/vstakhov/libucl.svg?branch=master)](https://travis-ci.org/vstakhov/libucl)[![Coverity](https://scan.coverity.com/projects/4138/badge.svg)](https://scan.coverity.com/projects/4138)[![Coverage Status](https://coveralls.io/repos/github/vstakhov/libucl/badge.svg?branch=master)](https://coveralls.io/github/vstakhov/libucl?branch=master) **Table of Contents** *generated with [DocToc](http://doctoc.herokuapp.com/)* diff --git a/contrib/libucl/configure.ac b/contrib/libucl/configure.ac index 73d322717cbc..722def8c85b4 100644 --- a/contrib/libucl/configure.ac +++ b/contrib/libucl/configure.ac @@ -1,7 +1,7 @@ m4_define([maj_ver], [0]) -m4_define([med_ver], [7]) -m4_define([min_ver], [3]) -m4_define([so_version], [5:0:2]) +m4_define([med_ver], [8]) +m4_define([min_ver], [0]) +m4_define([so_version], [6:0:0]) m4_define([ucl_version], [maj_ver.med_ver.min_ver]) AC_INIT([libucl],[ucl_version],[https://github.com/vstakhov/libucl],[libucl]) @@ -173,6 +173,8 @@ AC_LINK_IFELSE([ AC_MSG_WARN([Libucl references could be thread-unsafe because atomic builtins are missing]) ]) +AX_CODE_COVERAGE + AC_CONFIG_FILES(Makefile \ src/Makefile \ lua/Makefile diff --git a/contrib/libucl/include/ucl++.h b/contrib/libucl/include/ucl++.h index 87d2041a60ec..00297e7069aa 100644 --- a/contrib/libucl/include/ucl++.h +++ b/contrib/libucl/include/ucl++.h @@ -26,7 +26,6 @@ #include #include #include -#include #include "ucl.h" @@ -120,18 +119,19 @@ class Ucl final { it = std::shared_ptr(ucl_object_iterate_new (obj.obj.get()), ucl_iter_deleter()); cur.reset (new Ucl(ucl_object_iterate_safe (it.get(), true))); + if (!*cur) { + it.reset (); + cur.reset (); + } } const_iterator() {} - const_iterator(const const_iterator &other) { - it = other.it; - } + const_iterator(const const_iterator &other) = delete; + const_iterator(const_iterator &&other) = default; ~const_iterator() {} - const_iterator& operator=(const const_iterator &other) { - it = other.it; - return *this; - } + const_iterator& operator=(const const_iterator &other) = delete; + const_iterator& operator=(const_iterator &&other) = default; bool operator==(const const_iterator &other) const { @@ -264,45 +264,51 @@ class Ucl final { return res; } - double number_value () const + double number_value (const double default_val = 0.0) const { - if (obj) { - return ucl_object_todouble (obj.get()); + double res; + + if (ucl_object_todouble_safe(obj.get(), &res)) { + return res; } - return 0.0; + return default_val; } - int64_t int_value () const + int64_t int_value (const int64_t default_val = 0) const { - if (obj) { - return ucl_object_toint (obj.get()); + int64_t res; + + if (ucl_object_toint_safe(obj.get(), &res)) { + return res; } - return 0; + return default_val; } - bool bool_value () const + bool bool_value (const bool default_val = false) const { - if (obj) { - return ucl_object_toboolean (obj.get()); + bool res; + + if (ucl_object_toboolean_safe(obj.get(), &res)) { + return res; } - return false; + return default_val; } - const std::string string_value () const + const std::string string_value (const std::string& default_val = "") const { - std::string res; + const char* res = nullptr; - if (obj) { - res.assign (ucl_object_tostring (obj.get())); + if (ucl_object_tostring_safe(obj.get(), &res)) { + return res; } - return res; + return default_val; } - const Ucl operator[] (size_t i) const + const Ucl at (size_t i) const { if (type () == UCL_ARRAY) { return Ucl (ucl_array_find_index (obj.get(), i)); @@ -311,15 +317,25 @@ class Ucl final { return Ucl (nullptr); } - const Ucl operator[](const std::string &key) const + const Ucl lookup (const std::string &key) const { if (type () == UCL_OBJECT) { - return Ucl (ucl_object_find_keyl (obj.get(), + return Ucl (ucl_object_lookup_len (obj.get(), key.data (), key.size ())); } return Ucl (nullptr); } + + inline const Ucl operator[] (size_t i) const + { + return at(i); + } + + inline const Ucl operator[](const std::string &key) const + { + return lookup(key); + } // Serialize. void dump (std::string &out, ucl_emitter_t type = UCL_EMIT_JSON) const { @@ -328,7 +344,7 @@ class Ucl final { cbdata = Ucl::default_emit_funcs(); cbdata.ud = reinterpret_cast(&out); - ucl_object_emit_full (obj.get(), type, &cbdata); + ucl_object_emit_full (obj.get(), type, &cbdata, nullptr); } std::string dump (ucl_emitter_t type = UCL_EMIT_JSON) const @@ -375,6 +391,12 @@ class Ucl final { std::istreambuf_iterator()), err); } + Ucl& operator= (Ucl rhs) + { + obj.swap (rhs.obj); + return *this; + } + bool operator== (const Ucl &rhs) const { return ucl_object_compare (obj.get(), rhs.obj.get ()) == 0; @@ -388,7 +410,7 @@ class Ucl final { bool operator> (const Ucl &rhs) const { return (rhs < *this); } bool operator>= (const Ucl &rhs) const { return !(*this < rhs); } - operator bool () const + explicit operator bool () const { if (!obj || type() == UCL_NULL) { return false; diff --git a/contrib/libucl/include/ucl.h b/contrib/libucl/include/ucl.h index 82a1fd1880f3..024f5dd8fd35 100644 --- a/contrib/libucl/include/ucl.h +++ b/contrib/libucl/include/ucl.h @@ -107,7 +107,8 @@ typedef enum ucl_error { UCL_ENESTED, /**< Input has too many recursion levels */ UCL_EMACRO, /**< Error processing a macro */ UCL_EINTERNAL, /**< Internal unclassified error */ - UCL_ESSL /**< SSL error */ + UCL_ESSL, /**< SSL error */ + UCL_EMERGE /**< A merge error occured */ } ucl_error_t; /** @@ -147,11 +148,13 @@ typedef enum ucl_emitter { * UCL still has to perform copying implicitly. */ typedef enum ucl_parser_flags { - UCL_PARSER_DEFAULT = 0x0, /**< No special flags */ - UCL_PARSER_KEY_LOWERCASE = 0x1, /**< Convert all keys to lower case */ - UCL_PARSER_ZEROCOPY = 0x2, /**< Parse input in zero-copy mode if possible */ - UCL_PARSER_NO_TIME = 0x4, /**< Do not parse time and treat time values as strings */ - UCL_PARSER_NO_IMPLICIT_ARRAYS = 0x8 /** Create explicit arrays instead of implicit ones */ + UCL_PARSER_DEFAULT = 0, /**< No special flags */ + UCL_PARSER_KEY_LOWERCASE = (1 << 0), /**< Convert all keys to lower case */ + UCL_PARSER_ZEROCOPY = (1 << 1), /**< Parse input in zero-copy mode if possible */ + UCL_PARSER_NO_TIME = (1 << 2), /**< Do not parse time and treat time values as strings */ + UCL_PARSER_NO_IMPLICIT_ARRAYS = (1 << 3), /** Create explicit arrays instead of implicit ones */ + UCL_PARSER_SAVE_COMMENTS = (1 << 4), /** Save comments in the parser context */ + UCL_PARSER_DISABLE_MACRO = (1 << 5) /** Treat macros as comments */ } ucl_parser_flags_t; /** @@ -159,17 +162,17 @@ typedef enum ucl_parser_flags { */ typedef enum ucl_string_flags { UCL_STRING_RAW = 0x0, /**< Treat string as is */ - UCL_STRING_ESCAPE = 0x1, /**< Perform JSON escape */ - UCL_STRING_TRIM = 0x2, /**< Trim leading and trailing whitespaces */ - UCL_STRING_PARSE_BOOLEAN = 0x4, /**< Parse passed string and detect boolean */ - UCL_STRING_PARSE_INT = 0x8, /**< Parse passed string and detect integer number */ - UCL_STRING_PARSE_DOUBLE = 0x10, /**< Parse passed string and detect integer or float number */ - UCL_STRING_PARSE_TIME = 0x20, /**< Parse time strings */ + UCL_STRING_ESCAPE = (1 << 0), /**< Perform JSON escape */ + UCL_STRING_TRIM = (1 << 1), /**< Trim leading and trailing whitespaces */ + UCL_STRING_PARSE_BOOLEAN = (1 << 2), /**< Parse passed string and detect boolean */ + UCL_STRING_PARSE_INT = (1 << 3), /**< Parse passed string and detect integer number */ + UCL_STRING_PARSE_DOUBLE = (1 << 4), /**< Parse passed string and detect integer or float number */ + UCL_STRING_PARSE_TIME = (1 << 5), /**< Parse time strings */ UCL_STRING_PARSE_NUMBER = UCL_STRING_PARSE_INT|UCL_STRING_PARSE_DOUBLE|UCL_STRING_PARSE_TIME, /**< Parse passed string and detect number */ UCL_STRING_PARSE = UCL_STRING_PARSE_BOOLEAN|UCL_STRING_PARSE_NUMBER, /**< Parse passed string (and detect booleans and numbers) */ - UCL_STRING_PARSE_BYTES = 0x40 /**< Treat numbers as bytes */ + UCL_STRING_PARSE_BYTES = (1 << 6) /**< Treat numbers as bytes */ } ucl_string_flags_t; /** @@ -286,10 +289,12 @@ UCL_EXTERN ucl_object_t* ucl_object_new_full (ucl_type_t type, unsigned priority /** * Create new object with userdata dtor * @param dtor destructor function + * @param emitter emitter for userdata + * @param ptr opaque pointer * @return new object */ UCL_EXTERN ucl_object_t* ucl_object_new_userdata (ucl_userdata_dtor dtor, - ucl_userdata_emitter emitter) UCL_WARN_UNUSED_RESULT; + ucl_userdata_emitter emitter, void *ptr) UCL_WARN_UNUSED_RESULT; /** * Perform deep copy of an object copying everything @@ -305,6 +310,21 @@ UCL_EXTERN ucl_object_t * ucl_object_copy (const ucl_object_t *other) */ UCL_EXTERN ucl_type_t ucl_object_type (const ucl_object_t *obj); +/** + * Converts ucl object type to its string representation + * @param type type of object + * @return constant string describing type + */ +UCL_EXTERN const char * ucl_object_type_to_string (ucl_type_t type); + +/** + * Converts string that represents ucl type to real ucl type enum + * @param input C string with name of type + * @param res resulting target + * @return true if `input` is a name of type stored in `res` + */ +UCL_EXTERN bool ucl_object_string_to_type (const char *input, ucl_type_t *res); + /** * Convert any string to an ucl object making the specified transformations * @param str fixed size or NULL terminated string @@ -642,8 +662,9 @@ UCL_EXTERN const char* ucl_object_tolstring (const ucl_object_t *obj, size_t *tl * @param key key to search * @return object matching the specified key or NULL if key was not found */ -UCL_EXTERN const ucl_object_t* ucl_object_find_key (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t* ucl_object_lookup (const ucl_object_t *obj, const char *key); +#define ucl_object_find_key ucl_object_lookup /** * Return object identified by a key in the specified object, if the first key is @@ -655,8 +676,9 @@ UCL_EXTERN const ucl_object_t* ucl_object_find_key (const ucl_object_t *obj, * @param ... list of alternative keys to search (NULL terminated) * @return object matching the specified key or NULL if key was not found */ -UCL_EXTERN const ucl_object_t* ucl_object_find_any_key (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t* ucl_object_lookup_any (const ucl_object_t *obj, const char *key, ...); +#define ucl_object_find_any_key ucl_object_lookup_any /** * Return object identified by a fixed size key in the specified object @@ -665,8 +687,9 @@ UCL_EXTERN const ucl_object_t* ucl_object_find_any_key (const ucl_object_t *obj, * @param klen length of a key * @return object matching the specified key or NULL if key was not found */ -UCL_EXTERN const ucl_object_t* ucl_object_find_keyl (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t* ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen); +#define ucl_object_find_keyl ucl_object_lookup_len /** * Return object identified by dot notation string @@ -674,8 +697,9 @@ UCL_EXTERN const ucl_object_t* ucl_object_find_keyl (const ucl_object_t *obj, * @param path dot.notation.path to the path to lookup. May use numeric .index on arrays * @return object matched the specified path or NULL if path is not found */ -UCL_EXTERN const ucl_object_t *ucl_lookup_path (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t *ucl_object_lookup_path (const ucl_object_t *obj, const char *path); +#define ucl_lookup_path ucl_object_lookup_path /** * Return object identified by object notation string using arbitrary delimiter @@ -684,8 +708,9 @@ UCL_EXTERN const ucl_object_t *ucl_lookup_path (const ucl_object_t *obj, * @param sep the sepatorator to use in place of . (incase keys have . in them) * @return object matched the specified path or NULL if path is not found */ -UCL_EXTERN const ucl_object_t *ucl_lookup_path_char (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t *ucl_object_lookup_path_char (const ucl_object_t *obj, const char *path, char sep); +#define ucl_lookup_path_char ucl_object_lookup_path_char /** * Returns a key of an object as a NULL terminated string @@ -734,6 +759,19 @@ UCL_EXTERN void ucl_object_unref (ucl_object_t *obj); UCL_EXTERN int ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2); +/** + * Compare objects `o1` and `o2` useful for sorting + * @param o1 the first object + * @param o2 the second object + * @return values >0, 0 and <0 if `o1` is more than, equal and less than `o2`. + * The order of comparison: + * 1) Type of objects + * 2) Size of objects + * 3) Content of objects + */ +UCL_EXTERN int ucl_object_compare_qsort (const ucl_object_t **o1, + const ucl_object_t **o2); + /** * Sort UCL array using `cmp` compare function * @param ar @@ -770,8 +808,9 @@ typedef void* ucl_object_iter_t; * while ((cur = ucl_iterate_object (obj, &it)) != NULL) ... * @return the next object or NULL */ -UCL_EXTERN const ucl_object_t* ucl_iterate_object (const ucl_object_t *obj, +UCL_EXTERN const ucl_object_t* ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values); +#define ucl_iterate_object ucl_object_iterate /** * Create new safe iterator for the specified object @@ -1040,34 +1079,34 @@ UCL_EXTERN ucl_object_t* ucl_parser_get_object (struct ucl_parser *parser); * @param parser parser object * @return error description */ -UCL_EXTERN const char *ucl_parser_get_error(struct ucl_parser *parser); +UCL_EXTERN const char *ucl_parser_get_error (struct ucl_parser *parser); /** * Get the code of the last error * @param parser parser object * @return error code */ -UCL_EXTERN int ucl_parser_get_error_code(struct ucl_parser *parser); +UCL_EXTERN int ucl_parser_get_error_code (struct ucl_parser *parser); /** * Get the current column number within parser * @param parser parser object * @return current column number */ -UCL_EXTERN unsigned ucl_parser_get_column(struct ucl_parser *parser); +UCL_EXTERN unsigned ucl_parser_get_column (struct ucl_parser *parser); /** * Get the current line number within parser * @param parser parser object * @return current line number */ -UCL_EXTERN unsigned ucl_parser_get_linenum(struct ucl_parser *parser); +UCL_EXTERN unsigned ucl_parser_get_linenum (struct ucl_parser *parser); /** * Clear the error in the parser * @param parser parser object */ -UCL_EXTERN void ucl_parser_clear_error(struct ucl_parser *parser); +UCL_EXTERN void ucl_parser_clear_error (struct ucl_parser *parser); /** * Free ucl parser object @@ -1075,6 +1114,42 @@ UCL_EXTERN void ucl_parser_clear_error(struct ucl_parser *parser); */ UCL_EXTERN void ucl_parser_free (struct ucl_parser *parser); +/** + * Get constant opaque pointer to comments structure for this parser. Increase + * refcount to prevent this object to be destroyed on parser's destruction + * @param parser parser structure + * @return ucl comments pointer or NULL + */ +UCL_EXTERN const ucl_object_t * ucl_parser_get_comments (struct ucl_parser *parser); + +/** + * Utility function to find a comment object for the specified object in the input + * @param comments comments object + * @param srch search object + * @return string comment enclosed in ucl_object_t + */ +UCL_EXTERN const ucl_object_t * ucl_comments_find (const ucl_object_t *comments, + const ucl_object_t *srch); + +/** + * Move comment from `from` object to `to` object + * @param comments comments object + * @param what source object + * @param whith destination object + * @return `true` if `from` has comment and it has been moved to `to` + */ +UCL_EXTERN bool ucl_comments_move (ucl_object_t *comments, + const ucl_object_t *from, const ucl_object_t *to); + +/** + * Adds a new comment for an object + * @param comments comments object + * @param obj object to add comment to + * @param comment string representation of a comment + */ +UCL_EXTERN void ucl_comments_add (ucl_object_t *comments, + const ucl_object_t *obj, const char *comment); + /** * Add new public key to parser for signatures check * @param parser parser object @@ -1083,7 +1158,8 @@ UCL_EXTERN void ucl_parser_free (struct ucl_parser *parser); * @param err if *err is NULL it is set to parser error * @return true if a key has been successfully added */ -UCL_EXTERN bool ucl_pubkey_add (struct ucl_parser *parser, const unsigned char *key, size_t len); +UCL_EXTERN bool ucl_parser_pubkey_add (struct ucl_parser *parser, + const unsigned char *key, size_t len); /** * Set FILENAME and CURDIR variables in parser @@ -1156,8 +1232,8 @@ struct ucl_emitter_context { unsigned int indent; /** Top level object */ const ucl_object_t *top; - /** The rest of context */ - unsigned char data[1]; + /** Optional comments */ + const ucl_object_t *comments; }; /** @@ -1187,11 +1263,13 @@ UCL_EXTERN unsigned char *ucl_object_emit_len (const ucl_object_t *obj, * @param emit_type if type is #UCL_EMIT_JSON then emit json, if type is * #UCL_EMIT_CONFIG then emit config like object * @param emitter a set of emitter functions + * @param comments optional comments for the parser * @return dump of an object (must be freed after using) or NULL in case of error */ UCL_EXTERN bool ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type, - struct ucl_emitter_functions *emitter); + struct ucl_emitter_functions *emitter, + const ucl_object_t *comments); /** * Start streamlined UCL object emitter @@ -1280,6 +1358,9 @@ enum ucl_schema_error_code { UCL_SCHEMA_MISSING_PROPERTY,/**< one or more missing properties */ UCL_SCHEMA_CONSTRAINT, /**< constraint found */ UCL_SCHEMA_MISSING_DEPENDENCY, /**< missing dependency */ + UCL_SCHEMA_EXTERNAL_REF_MISSING, /**< cannot fetch external ref */ + UCL_SCHEMA_EXTERNAL_REF_INVALID, /**< invalid external ref */ + UCL_SCHEMA_INTERNAL_ERROR, /**< something bad happened */ UCL_SCHEMA_UNKNOWN /**< generic error */ }; @@ -1303,6 +1384,37 @@ struct ucl_schema_error { UCL_EXTERN bool ucl_object_validate (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err); +/** + * Validate object `obj` using schema object `schema` and root schema at `root`. + * @param schema schema object + * @param obj object to validate + * @param root root schema object + * @param err error pointer, if this parameter is not NULL and error has been + * occured, then `err` is filled with the exact error definition. + * @return true if `obj` is valid using `schema` + */ +UCL_EXTERN bool ucl_object_validate_root (const ucl_object_t *schema, + const ucl_object_t *obj, + const ucl_object_t *root, + struct ucl_schema_error *err); + +/** + * Validate object `obj` using schema object `schema` and root schema at `root` + * using some external references provided. + * @param schema schema object + * @param obj object to validate + * @param root root schema object + * @param ext_refs external references (might be modified during validation) + * @param err error pointer, if this parameter is not NULL and error has been + * occured, then `err` is filled with the exact error definition. + * @return true if `obj` is valid using `schema` + */ +UCL_EXTERN bool ucl_object_validate_root_ext (const ucl_object_t *schema, + const ucl_object_t *obj, + const ucl_object_t *root, + ucl_object_t *ext_refs, + struct ucl_schema_error *err); + /** @} */ #ifdef __cplusplus diff --git a/contrib/libucl/lua/lua_ucl.c b/contrib/libucl/lua/lua_ucl.c index b6162b432f59..bf80810d3e07 100644 --- a/contrib/libucl/lua/lua_ucl.c +++ b/contrib/libucl/lua/lua_ucl.c @@ -29,6 +29,7 @@ #include "ucl_internal.h" #include "lua_ucl.h" #include +#include /*** * @module ucl @@ -149,14 +150,14 @@ ucl_object_lua_push_object (lua_State *L, const ucl_object_t *obj, } /* Optimize allocation by preallocation of table */ - while (ucl_iterate_object (obj, &it, true) != NULL) { + while (ucl_object_iterate (obj, &it, true) != NULL) { nelt ++; } lua_createtable (L, 0, nelt); it = NULL; - while ((cur = ucl_iterate_object (obj, &it, true)) != NULL) { + while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { ucl_object_lua_push_element (L, ucl_object_key (cur), cur); } @@ -421,9 +422,7 @@ ucl_object_lua_fromelt (lua_State *L, int idx) fd->idx = luaL_ref (L, LUA_REGISTRYINDEX); obj = ucl_object_new_userdata (lua_ucl_userdata_dtor, - lua_ucl_userdata_emitter); - obj->type = UCL_USERDATA; - obj->value.ud = (void *)fd; + lua_ucl_userdata_emitter, (void *)fd); } } } @@ -514,6 +513,17 @@ lua_ucl_object_get (lua_State *L, int index) return *((ucl_object_t **) luaL_checkudata(L, index, OBJECT_META)); } +static void +lua_ucl_push_opaque (lua_State *L, ucl_object_t *obj) +{ + ucl_object_t **pobj; + + pobj = lua_newuserdata (L, sizeof (*pobj)); + *pobj = obj; + luaL_getmetatable (L, OBJECT_META); + lua_setmetatable (L, -2); +} + /*** * @method parser:parse_file(name) * Parse UCL object from file. @@ -629,17 +639,14 @@ static int lua_ucl_parser_get_object_wrapped (lua_State *L) { struct ucl_parser *parser; - ucl_object_t *obj, **pobj; + ucl_object_t *obj; int ret = 1; parser = lua_ucl_parser_get (L, 1); obj = ucl_parser_get_object (parser); if (obj != NULL) { - pobj = lua_newuserdata (L, sizeof (*pobj)); - *pobj = obj; - luaL_getmetatable (L, OBJECT_META); - lua_setmetatable (L, -2); + lua_ucl_push_opaque (L, obj); } else { lua_pushnil (L); @@ -806,19 +813,21 @@ lua_ucl_object_tostring (lua_State *L) } /*** - * @method object:validate(schema, path) + * @method object:validate(schema[, path[, ext_refs]]) * Validates the given ucl object using schema object represented as another * opaque ucl object. You can also specify path in the form `#/path/def` to * specify the specific schema element to perform validation. * * @param {ucl.object} schema schema object * @param {string} path optional path for validation procedure - * @return {result,err} two values: boolean result and the corresponding error + * @return {result,err} two values: boolean result and the corresponding + * error, if `ext_refs` are also specified, then they are returned as opaque + * ucl object as {result,err,ext_refs} */ static int lua_ucl_object_validate (lua_State *L) { - ucl_object_t *obj, *schema; + ucl_object_t *obj, *schema, *ext_refs = NULL; const ucl_object_t *schema_elt; bool res = false; struct ucl_schema_error err; @@ -828,15 +837,30 @@ lua_ucl_object_validate (lua_State *L) schema = lua_ucl_object_get (L, 2); if (schema && obj && ucl_object_type (schema) == UCL_OBJECT) { - if (lua_gettop (L) > 2 && lua_type (L, 3) == LUA_TSTRING) { - path = lua_tostring (L, 3); - if (path[0] == '#') { - path ++; + if (lua_gettop (L) > 2) { + if (lua_type (L, 3) == LUA_TSTRING) { + path = lua_tostring (L, 3); + if (path[0] == '#') { + path++; + } + } + else if (lua_type (L, 3) == LUA_TUSERDATA || lua_type (L, 3) == + LUA_TTABLE) { + /* External refs */ + ext_refs = lua_ucl_object_get (L, 3); + } + + if (lua_gettop (L) > 3) { + if (lua_type (L, 4) == LUA_TUSERDATA || lua_type (L, 4) == + LUA_TTABLE) { + /* External refs */ + ext_refs = lua_ucl_object_get (L, 4); + } } } if (path) { - schema_elt = ucl_lookup_path_char (schema, path, '/'); + schema_elt = ucl_object_lookup_path_char (schema, path, '/'); } else { /* Use the top object */ @@ -844,26 +868,33 @@ lua_ucl_object_validate (lua_State *L) } if (schema_elt) { - res = ucl_object_validate (schema_elt, obj, &err); + res = ucl_object_validate_root_ext (schema_elt, obj, schema, + ext_refs, &err); if (res) { lua_pushboolean (L, res); lua_pushnil (L); + + if (ext_refs) { + lua_ucl_push_opaque (L, ext_refs); + } } else { lua_pushboolean (L, res); lua_pushfstring (L, "validation error: %s", err.msg); + + if (ext_refs) { + lua_ucl_push_opaque (L, ext_refs); + } } } else { lua_pushboolean (L, res); - if (path) { - lua_pushfstring (L, "cannot find the requested path: %s", path); - } - else { - /* Should not be reached */ - lua_pushstring (L, "unknown error"); + lua_pushfstring (L, "cannot find the requested path: %s", path); + + if (ext_refs) { + lua_ucl_push_opaque (L, ext_refs); } } } @@ -872,6 +903,10 @@ lua_ucl_object_validate (lua_State *L) lua_pushstring (L, "invalid object or schema"); } + if (ext_refs) { + return 3; + } + return 2; } diff --git a/contrib/libucl/python/src/uclmodule.c b/contrib/libucl/python/src/uclmodule.c index d63b73aad9d8..bc13c704b9c1 100644 --- a/contrib/libucl/python/src/uclmodule.c +++ b/contrib/libucl/python/src/uclmodule.c @@ -2,65 +2,63 @@ #include #include -static PyObject* -_basic_ucl_type(ucl_object_t const * const obj) { - if (obj->type == UCL_INT) { - return Py_BuildValue("L", (long long)ucl_object_toint (obj)); - } - else if (obj->type == UCL_FLOAT) { - return Py_BuildValue("d", ucl_object_todouble (obj)); - } - else if (obj->type == UCL_STRING) { - return Py_BuildValue("s", ucl_object_tostring (obj)); - } - else if (obj->type == UCL_BOOLEAN) { - // maybe used 'p' here? - return Py_BuildValue("s", ucl_object_tostring_forced (obj)); - } - else if (obj->type == UCL_TIME) { - return Py_BuildValue("d", ucl_object_todouble (obj)); +static PyObject * +_basic_ucl_type (ucl_object_t const *obj) +{ + switch (obj->type) { + case UCL_INT: + return Py_BuildValue ("L", (long long)ucl_object_toint (obj)); + case UCL_FLOAT: + return Py_BuildValue ("d", ucl_object_todouble (obj)); + case UCL_STRING: + return Py_BuildValue ("s", ucl_object_tostring (obj)); + case UCL_BOOLEAN: + return ucl_object_toboolean (obj) ? Py_True : Py_False; + case UCL_TIME: + return Py_BuildValue ("d", ucl_object_todouble (obj)); } return NULL; } -static PyObject* -_iterate_valid_ucl(ucl_object_t const * obj) { +static PyObject * +_iterate_valid_ucl (ucl_object_t const *obj) +{ const ucl_object_t *tmp; ucl_object_iter_t it = NULL; tmp = obj; - while ((obj = ucl_iterate_object (tmp, &it, false))) { - - PyObject* val; + while ((obj = ucl_object_iterate (tmp, &it, false))) { + PyObject *val; val = _basic_ucl_type(obj); if (!val) { - PyObject* key = NULL; + PyObject *key = NULL; + if (obj->key != NULL) { key = Py_BuildValue("s", ucl_object_key(obj)); } - PyObject* ret; - ret = PyDict_New(); if (obj->type == UCL_OBJECT) { - val = PyDict_New(); const ucl_object_t *cur; ucl_object_iter_t it_obj = NULL; - while ((cur = ucl_iterate_object (obj, &it_obj, true))) { - PyObject* keyobj = Py_BuildValue("s",ucl_object_key(cur)); + + val = PyDict_New(); + + while ((cur = ucl_object_iterate (obj, &it_obj, true))) { + PyObject *keyobj = Py_BuildValue("s",ucl_object_key(cur)); PyDict_SetItem(val, keyobj, _iterate_valid_ucl(cur)); } - } - else if (obj->type == UCL_ARRAY) { - val = PyList_New(0); + } else if (obj->type == UCL_ARRAY) { const ucl_object_t *cur; ucl_object_iter_t it_obj = NULL; - while ((cur = ucl_iterate_object (obj, &it_obj, true))) { + + val = PyList_New(0); + + while ((cur = ucl_object_iterate (obj, &it_obj, true))) { PyList_Append(val, _iterate_valid_ucl(cur)); } - } - else if (obj->type == UCL_USERDATA) { + } else if (obj->type == UCL_USERDATA) { // XXX: this should be // PyBytes_FromStringAndSize; where is the // length from? @@ -74,13 +72,13 @@ _iterate_valid_ucl(ucl_object_t const * obj) { return NULL; } -static PyObject* -_internal_load_ucl(char* uclstr) { - PyObject* ret; - +static PyObject * +_internal_load_ucl (char *uclstr) +{ + PyObject *ret; struct ucl_parser *parser = ucl_parser_new (UCL_PARSER_NO_TIME); - bool r = ucl_parser_add_string(parser, uclstr, 0); + if (r) { if (ucl_parser_get_error (parser)) { PyErr_SetString(PyExc_ValueError, ucl_parser_get_error(parser)); @@ -88,13 +86,13 @@ _internal_load_ucl(char* uclstr) { ret = NULL; goto return_with_parser; } else { - ucl_object_t* uclobj = ucl_parser_get_object(parser); + ucl_object_t *uclobj = ucl_parser_get_object(parser); ret = _iterate_valid_ucl(uclobj); ucl_object_unref(uclobj); goto return_with_parser; } - - } else { + } + else { PyErr_SetString(PyExc_ValueError, ucl_parser_get_error (parser)); ret = NULL; goto return_with_parser; @@ -106,36 +104,151 @@ _internal_load_ucl(char* uclstr) { } static PyObject* -ucl_load(PyObject *self, PyObject *args) { - char* uclstr; +ucl_load (PyObject *self, PyObject *args) +{ + char *uclstr; + if (PyArg_ParseTuple(args, "z", &uclstr)) { if (!uclstr) { Py_RETURN_NONE; } + return _internal_load_ucl(uclstr); } + return NULL; } -static PyObject* -ucl_validate(PyObject *self, PyObject *args) { - char *uclstr, *schema; +static ucl_object_t * +_iterate_python (PyObject *obj) +{ + if (obj == Py_None) { + return ucl_object_new(); + } else if (PyBool_Check (obj)) { + return ucl_object_frombool (obj == Py_True); + } else if (PyInt_Check (obj)) { + return ucl_object_fromint (PyInt_AsLong (obj)); + } else if (PyFloat_Check (obj)) { + return ucl_object_fromdouble (PyFloat_AsDouble (obj)); + } else if (PyString_Check (obj)) { + return ucl_object_fromstring (PyString_AsString (obj)); + // } else if (PyDateTime_Check (obj)) { + } + else if (PyDict_Check(obj)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + ucl_object_t *top, *elm; + + top = ucl_object_typed_new (UCL_OBJECT); + + while (PyDict_Next(obj, &pos, &key, &value)) { + elm = _iterate_python(value); + ucl_object_insert_key (top, elm, PyString_AsString(key), 0, true); + } + + return top; + } + else if (PySequence_Check(obj)) { + PyObject *value; + Py_ssize_t len, pos; + ucl_object_t *top, *elm; + + len = PySequence_Length(obj); + top = ucl_object_typed_new (UCL_ARRAY); + + for (pos = 0; pos < len; pos++) { + value = PySequence_GetItem(obj, pos); + elm = _iterate_python(value); + ucl_array_append(top, elm); + } + + return top; + } + else { + PyErr_SetString(PyExc_TypeError, "Unhandled object type"); + return NULL; + } + + return NULL; +} + +static PyObject * +ucl_dump (PyObject *self, PyObject *args) +{ + PyObject *obj; + ucl_emitter_t emitter; + ucl_object_t *root = NULL; + + emitter = UCL_EMIT_CONFIG; + + if (!PyArg_ParseTuple(args, "O|i", &obj, &emitter)) { + PyErr_SetString(PyExc_TypeError, "Unhandled object type"); + return NULL; + } + + if (emitter >= UCL_EMIT_MAX) { + PyErr_SetString(PyExc_TypeError, "Invalid emitter type"); + return NULL; + } + + if (obj == Py_None) { + Py_RETURN_NONE; + } + + if (!PyDict_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "Argument must be dict"); + return NULL; + } + + root = _iterate_python(obj); + if (root) { + PyObject *ret; + char *buf; + + buf = (char *) ucl_object_emit (root, emitter); + ucl_object_unref (root); + ret = PyString_FromString (buf); + free(buf); + + return ret; + } + + return NULL; +} + +static PyObject * +ucl_validate (PyObject *self, PyObject *args) +{ + char *uclstr, *schema; + if (PyArg_ParseTuple(args, "zz", &uclstr, &schema)) { if (!uclstr || !schema) { Py_RETURN_NONE; } + PyErr_SetString(PyExc_NotImplementedError, "schema validation is not yet supported"); - return NULL; } + return NULL; } static PyMethodDef uclMethods[] = { {"load", ucl_load, METH_VARARGS, "Load UCL from stream"}, + {"dump", ucl_dump, METH_VARARGS, "Dump UCL to stream"}, {"validate", ucl_validate, METH_VARARGS, "Validate ucl stream against schema"}, {NULL, NULL, 0, NULL} }; +static void +init_macros(PyObject *mod) +{ + PyModule_AddIntMacro(mod, UCL_EMIT_JSON); + PyModule_AddIntMacro(mod, UCL_EMIT_JSON_COMPACT); + PyModule_AddIntMacro(mod, UCL_EMIT_CONFIG); + PyModule_AddIntMacro(mod, UCL_EMIT_YAML); + PyModule_AddIntMacro(mod, UCL_EMIT_MSGPACK); +} + #if PY_MAJOR_VERSION >= 3 static struct PyModuleDef uclmodule = { PyModuleDef_HEAD_INIT, @@ -146,11 +259,17 @@ static struct PyModuleDef uclmodule = { }; PyMODINIT_FUNC -PyInit_ucl(void) { - return PyModule_Create(&uclmodule); +PyInit_ucl (void) +{ + PyObject *mod = PyModule_Create (&uclmodule); + init_macros (mod); + + return mod; } #else -void initucl(void) { - Py_InitModule("ucl", uclMethods); +void initucl (void) +{ + PyObject *mod = Py_InitModule ("ucl", uclMethods); + init_macros (mod); } #endif diff --git a/contrib/libucl/python/test_uclmodule.py b/contrib/libucl/python/test_uclmodule.py index de295dc2000f..0d774696aa4f 100755 --- a/contrib/libucl/python/test_uclmodule.py +++ b/contrib/libucl/python/test_uclmodule.py @@ -37,6 +37,14 @@ def test_str(self): def test_float(self): self.assertEqual(ucl.load("a : 1.1"), {"a" : 1.1}) + def test_boolean(self): + totest = ( + "a : True;" \ + "b : False" + ) + correct = {"a" : True, "b" : False} + self.assertEqual(ucl.load(totest), correct) + def test_empty_ucl(self): r = ucl.load("{}") self.assertEqual(r, {}) @@ -86,9 +94,9 @@ def test_every_type(self): 'key7': '0xreadbeef', 'key8': -1e-10, 'key9': 1, - 'key10': 'true', - 'key11': 'false', - 'key12': 'true', + 'key10': True, + 'key11': False, + 'key12': True, } self.assertEqual(ucl.load(totest), correct) @@ -96,5 +104,45 @@ def test_validation_useless(self): with self.assertRaises(NotImplementedError): ucl.validate("","") + +class TestUclDump(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + ucl.dump() + + def test_multi_args(self): + with self.assertRaises(TypeError): + ucl.dump(0, 0) + + def test_none(self): + self.assertEqual(ucl.dump(None), None) + + def test_int(self): + self.assertEqual(ucl.dump({ "a" : 1 }), "a = 1;\n") + + def test_nested_int(self): + self.assertEqual(ucl.dump({ "a" : { "b" : 1 } }), "a {\n b = 1;\n}\n") + + def test_int_array(self): + self.assertEqual(ucl.dump({ "a" : [1,2,3,4]}), "a [\n 1,\n 2,\n 3,\n 4,\n]\n") + + def test_str(self): + self.assertEqual(ucl.dump({"a" : "b"}), "a = \"b\";\n") + + def test_float(self): + self.assertEqual(ucl.dump({"a" : 1.1}), "a = 1.100000;\n") + + def test_boolean(self): + totest = {"a" : True, "b" : False} + correct = "a = true;\nb = false;\n" + self.assertEqual(ucl.dump(totest), correct) + + def test_empty_ucl(self): + self.assertEqual(ucl.dump({}), "") + + def test_json(self): + self.assertEqual(ucl.dump({ "a" : 1, "b": "bleh;" }, ucl.UCL_EMIT_JSON), '{\n "a": 1,\n "b": "bleh;"\n}') + + if __name__ == '__main__': unittest.main() diff --git a/contrib/libucl/src/ucl_emitter.c b/contrib/libucl/src/ucl_emitter.c index 8bfbf09b8bd7..a15cd08cfb98 100644 --- a/contrib/libucl/src/ucl_emitter.c +++ b/contrib/libucl/src/ucl_emitter.c @@ -268,7 +268,7 @@ ucl_emitter_common_start_array (struct ucl_emitter_context *ctx, if (obj->type == UCL_ARRAY) { /* explicit array */ - while ((cur = ucl_iterate_object (obj, &iter, true)) != NULL) { + while ((cur = ucl_object_iterate (obj, &iter, true)) != NULL) { ucl_emitter_common_elt (ctx, cur, first, false, compact); first = false; } @@ -362,6 +362,7 @@ ucl_emitter_common_elt (struct ucl_emitter_context *ctx, const struct ucl_emitter_functions *func = ctx->func; bool flag; struct ucl_object_userdata *ud; + const ucl_object_t *comment = NULL, *cur_comment; const char *ud_out = ""; if (ctx->id != UCL_EMIT_CONFIG && !first) { @@ -379,6 +380,25 @@ ucl_emitter_common_elt (struct ucl_emitter_context *ctx, ucl_add_tabs (func, ctx->indent, compact); + if (ctx->comments && ctx->id == UCL_EMIT_CONFIG) { + comment = ucl_object_lookup_len (ctx->comments, (const char *)&obj, + sizeof (void *)); + + if (comment) { + if (!(comment->flags & UCL_OBJECT_INHERITED)) { + DL_FOREACH (comment, cur_comment) { + func->ucl_emitter_append_len (cur_comment->value.sv, + cur_comment->len, + func->ud); + func->ucl_emitter_append_character ('\n', 1, func->ud); + ucl_add_tabs (func, ctx->indent, compact); + } + + comment = NULL; + } + } + } + switch (obj->type) { case UCL_INT: ucl_emitter_print_key (print_key, ctx, obj, compact); @@ -438,6 +458,19 @@ ucl_emitter_common_elt (struct ucl_emitter_context *ctx, ucl_emitter_finish_object (ctx, obj, compact, !print_key); break; } + + if (comment) { + DL_FOREACH (comment, cur_comment) { + func->ucl_emitter_append_len (cur_comment->value.sv, + cur_comment->len, + func->ud); + func->ucl_emitter_append_character ('\n', 1, func->ud); + + if (cur_comment->next) { + ucl_add_tabs (func, ctx->indent, compact); + } + } + } } /* @@ -518,7 +551,7 @@ ucl_emit_msgpack_elt (struct ucl_emitter_context *ctx, ucl_emit_msgpack_start_obj (ctx, obj, print_key); it = NULL; - while ((cur = ucl_iterate_object (obj, &it, true)) != NULL) { + while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { LL_FOREACH (cur, celt) { ucl_emit_msgpack_elt (ctx, celt, false, true); /* XXX: @@ -537,7 +570,7 @@ ucl_emit_msgpack_elt (struct ucl_emitter_context *ctx, ucl_emit_msgpack_start_array (ctx, obj, print_key); it = NULL; - while ((cur = ucl_iterate_object (obj, &it, true)) != NULL) { + while ((cur = ucl_object_iterate (obj, &it, true)) != NULL) { ucl_emit_msgpack_elt (ctx, cur, false, false); } @@ -605,10 +638,10 @@ ucl_object_emit_len (const ucl_object_t *obj, enum ucl_emitter emit_type, } func = ucl_object_emit_memory_funcs ((void **)&res); - s = func->ud; if (func != NULL) { - ucl_object_emit_full (obj, emit_type, func); + s = func->ud; + ucl_object_emit_full (obj, emit_type, func, NULL); if (outlen != NULL) { *outlen = s->i; @@ -622,7 +655,8 @@ ucl_object_emit_len (const ucl_object_t *obj, enum ucl_emitter emit_type, bool ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type, - struct ucl_emitter_functions *emitter) + struct ucl_emitter_functions *emitter, + const ucl_object_t *comments) { const struct ucl_emitter_context *ctx; struct ucl_emitter_context my_ctx; @@ -634,6 +668,7 @@ ucl_object_emit_full (const ucl_object_t *obj, enum ucl_emitter emit_type, my_ctx.func = emitter; my_ctx.indent = 0; my_ctx.top = obj; + my_ctx.comments = comments; my_ctx.ops->ucl_emitter_write_elt (&my_ctx, obj, true, false); res = true; diff --git a/contrib/libucl/src/ucl_emitter_streamline.c b/contrib/libucl/src/ucl_emitter_streamline.c index ff27c88244a5..a7178c5d74b0 100644 --- a/contrib/libucl/src/ucl_emitter_streamline.c +++ b/contrib/libucl/src/ucl_emitter_streamline.c @@ -38,12 +38,20 @@ struct ucl_emitter_streamline_stack { struct ucl_emitter_context_streamline { /* Inherited from the main context */ + /** Name of emitter (e.g. json, compact_json) */ const char *name; + /** Unique id (e.g. UCL_EMIT_JSON for standard emitters */ int id; + /** A set of output functions */ const struct ucl_emitter_functions *func; + /** A set of output operations */ const struct ucl_emitter_operations *ops; - unsigned int ident; + /** Current amount of indent tabs */ + unsigned int indent; + /** Top level object */ const ucl_object_t *top; + /** Optional comments */ + const ucl_object_t *comments; /* Streamline specific fields */ struct ucl_emitter_streamline_stack *containers; diff --git a/contrib/libucl/src/ucl_hash.c b/contrib/libucl/src/ucl_hash.c index 7de8196fa014..c54fba7c90a5 100644 --- a/contrib/libucl/src/ucl_hash.c +++ b/contrib/libucl/src/ucl_hash.c @@ -117,7 +117,7 @@ static inline int ucl_hash_equal (const ucl_object_t *k1, const ucl_object_t *k2) { if (k1->keylen == k2->keylen) { - return strncmp (k1->key, k2->key, k1->keylen) == 0; + return memcmp (k1->key, k2->key, k1->keylen) == 0; } return 0; @@ -216,7 +216,7 @@ static inline int ucl_hash_caseless_equal (const ucl_object_t *k1, const ucl_object_t *k2) { if (k1->keylen == k2->keylen) { - return strncasecmp (k1->key, k2->key, k1->keylen) == 0; + return memcmp (k1->key, k2->key, k1->keylen) == 0; } return 0; @@ -247,7 +247,7 @@ ucl_hash_create (bool ignore_case) return new; } -void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func *func) +void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func) { const ucl_object_t *cur, *tmp; diff --git a/contrib/libucl/src/ucl_hash.h b/contrib/libucl/src/ucl_hash.h index 64c83eac8bc1..92021e34075e 100644 --- a/contrib/libucl/src/ucl_hash.h +++ b/contrib/libucl/src/ucl_hash.h @@ -31,8 +31,8 @@ struct ucl_hash_node_s; typedef struct ucl_hash_node_s ucl_hash_node_t; -typedef int ucl_hash_cmp_func (const void* void_a, const void* void_b); -typedef void ucl_hash_free_func (void *ptr); +typedef int (*ucl_hash_cmp_func) (const void* void_a, const void* void_b); +typedef void (*ucl_hash_free_func) (void *ptr); typedef void* ucl_hash_iter_t; @@ -51,7 +51,7 @@ ucl_hash_t* ucl_hash_create (bool ignore_case); /** * Deinitializes the hashtable. */ -void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func *func); +void ucl_hash_destroy (ucl_hash_t* hashlin, ucl_hash_free_func func); /** * Inserts an element in the the hashtable. diff --git a/contrib/libucl/src/ucl_internal.h b/contrib/libucl/src/ucl_internal.h index 31d6b1361383..db8a12c408b1 100644 --- a/contrib/libucl/src/ucl_internal.h +++ b/contrib/libucl/src/ucl_internal.h @@ -211,6 +211,8 @@ struct ucl_parser { struct ucl_variable *variables; ucl_variable_handler var_handler; void *var_data; + ucl_object_t *comments; + ucl_object_t *last_comment; UT_string *err; }; @@ -523,6 +525,34 @@ void ucl_emitter_print_key_msgpack (bool print_key, struct ucl_emitter_context *ctx, const ucl_object_t *obj); +/** + * Fetch URL into a buffer + * @param url url to fetch + * @param buf pointer to buffer (must be freed by callee) + * @param buflen pointer to buffer length + * @param err pointer to error argument + * @param must_exist fail if cannot find a url + */ +bool ucl_fetch_url (const unsigned char *url, + unsigned char **buf, + size_t *buflen, + UT_string **err, + bool must_exist); + +/** + * Fetch a file and save results to the memory buffer + * @param filename filename to fetch + * @param len length of filename + * @param buf target buffer + * @param buflen target length + * @return + */ +bool ucl_fetch_file (const unsigned char *filename, + unsigned char **buf, + size_t *buflen, + UT_string **err, + bool must_exist); + /** * Add new element to an object using the current merge strategy and priority * @param parser diff --git a/contrib/libucl/src/ucl_msgpack.c b/contrib/libucl/src/ucl_msgpack.c index e8ebfba8a35e..96f4809f892f 100644 --- a/contrib/libucl/src/ucl_msgpack.c +++ b/contrib/libucl/src/ucl_msgpack.c @@ -113,19 +113,19 @@ ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx, int64_t val) len = 1; buf[0] = mask_positive & val; } - else if (val <= 0xff) { + else if (val <= UINT8_MAX) { len = 2; buf[0] = uint8_ch; buf[1] = val & 0xff; } - else if (val <= 0xffff) { + else if (val <= UINT16_MAX) { uint16_t v = TO_BE16 (val); len = 3; buf[0] = uint16_ch; memcpy (&buf[1], &v, sizeof (v)); } - else if (val <= 0xffffffff) { + else if (val <= UINT32_MAX) { uint32_t v = TO_BE32 (val); len = 5; @@ -149,19 +149,20 @@ ucl_emitter_print_int_msgpack (struct ucl_emitter_context *ctx, int64_t val) len = 1; buf[0] = (mask_negative | uval) & 0xff; } - else if (uval <= 0xff) { + else if (uval <= INT8_MAX) { + uint8_t v = (uint8_t)val; len = 2; buf[0] = int8_ch; - buf[1] = (unsigned char)val; + buf[1] = v; } - else if (uval <= 0xffff) { + else if (uval <= INT16_MAX) { uint16_t v = TO_BE16 (val); len = 3; buf[0] = int16_ch; memcpy (&buf[1], &v, sizeof (v)); } - else if (uval <= 0xffffffff) { + else if (uval <= INT32_MAX) { uint32_t v = TO_BE32 (val); len = 5; @@ -750,7 +751,7 @@ ucl_msgpack_get_parser_from_type (unsigned char t) shift = CHAR_BIT - parsers[i].prefixlen; mask = parsers[i].prefix >> shift; - if (mask == (t >> shift)) { + if (mask == (((unsigned int)t) >> shift)) { return &parsers[i]; } } @@ -969,8 +970,8 @@ ucl_msgpack_consume (struct ucl_parser *parser) finish_array_value, error_state } state = read_type, next_state = error_state; - struct ucl_msgpack_parser *obj_parser; - uint64_t len; + struct ucl_msgpack_parser *obj_parser = NULL; + uint64_t len = 0; ssize_t ret, remain, keylen = 0; #ifdef MSGPACK_DEBUG_PARSER uint64_t i; @@ -1418,6 +1419,10 @@ ucl_msgpack_parse_int (struct ucl_parser *parser, const unsigned char *pos, size_t remain) { ucl_object_t *obj; + int8_t iv8; + int16_t iv16; + int32_t iv32; + int64_t iv64; if (len > remain) { return -1; @@ -1439,11 +1444,14 @@ ucl_msgpack_parse_int (struct ucl_parser *parser, len = 1; break; case msgpack_int8: - obj->value.iv = (signed char)*pos; + memcpy (&iv8, pos, sizeof (iv8)); + obj->value.iv = iv8; len = 1; break; case msgpack_int16: - obj->value.iv = FROM_BE16 (*(int16_t *)pos); + memcpy (&iv16, pos, sizeof (iv16)); + iv16 = FROM_BE16 (iv16); + obj->value.iv = iv16; len = 2; break; case msgpack_uint16: @@ -1451,7 +1459,9 @@ ucl_msgpack_parse_int (struct ucl_parser *parser, len = 2; break; case msgpack_int32: - obj->value.iv = FROM_BE32 (*(int32_t *)pos); + memcpy (&iv32, pos, sizeof (iv32)); + iv32 = FROM_BE32 (iv32); + obj->value.iv = iv32; len = 4; break; case msgpack_uint32: @@ -1459,7 +1469,9 @@ ucl_msgpack_parse_int (struct ucl_parser *parser, len = 4; break; case msgpack_int64: - obj->value.iv = FROM_BE64 (*(int64_t *)pos); + memcpy (&iv64, pos, sizeof (iv64)); + iv64 = FROM_BE64 (iv64); + obj->value.iv = iv64; len = 8; break; case msgpack_uint64: diff --git a/contrib/libucl/src/ucl_parser.c b/contrib/libucl/src/ucl_parser.c index 9bd41391ba36..0aaa4dd988db 100644 --- a/contrib/libucl/src/ucl_parser.c +++ b/contrib/libucl/src/ucl_parser.c @@ -89,6 +89,39 @@ ucl_set_err (struct ucl_parser *parser, int code, const char *str, UT_string **e parser->err_code = code; } +static void +ucl_save_comment (struct ucl_parser *parser, const char *begin, size_t len) +{ + ucl_object_t *nobj; + + if (len > 0 && begin != NULL) { + nobj = ucl_object_fromstring_common (begin, len, 0); + + if (parser->last_comment) { + /* We need to append data to an existing object */ + DL_APPEND (parser->last_comment, nobj); + } + else { + parser->last_comment = nobj; + } + } +} + +static void +ucl_attach_comment (struct ucl_parser *parser, ucl_object_t *obj, bool before) +{ + if (parser->last_comment) { + ucl_object_insert_key (parser->comments, parser->last_comment, + (const char *)&obj, sizeof (void *), true); + + if (before) { + parser->last_comment->flags |= UCL_OBJECT_INHERITED; + } + + parser->last_comment = NULL; + } +} + /** * Skip all comments from the current pos resolving nested and multiline comments * @param parser @@ -98,7 +131,7 @@ static bool ucl_skip_comments (struct ucl_parser *parser) { struct ucl_chunk *chunk = parser->chunks; - const unsigned char *p; + const unsigned char *p, *beg = NULL; int comments_nested = 0; bool quoted = false; @@ -108,9 +141,17 @@ ucl_skip_comments (struct ucl_parser *parser) if (chunk->remain > 0 && *p == '#') { if (parser->state != UCL_STATE_SCOMMENT && parser->state != UCL_STATE_MCOMMENT) { + beg = p; + while (p < chunk->end) { if (*p == '\n') { + if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { + ucl_save_comment (parser, beg, p - beg); + beg = NULL; + } + ucl_chunk_skipc (chunk, p); + goto start; } ucl_chunk_skipc (chunk, p); @@ -119,6 +160,7 @@ ucl_skip_comments (struct ucl_parser *parser) } else if (chunk->remain >= 2 && *p == '/') { if (p[1] == '*') { + beg = p; ucl_chunk_skipc (chunk, p); comments_nested ++; ucl_chunk_skipc (chunk, p); @@ -134,6 +176,11 @@ ucl_skip_comments (struct ucl_parser *parser) if (*p == '/') { comments_nested --; if (comments_nested == 0) { + if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { + ucl_save_comment (parser, beg, p - beg + 1); + beg = NULL; + } + ucl_chunk_skipc (chunk, p); goto start; } @@ -147,6 +194,7 @@ ucl_skip_comments (struct ucl_parser *parser) continue; } } + ucl_chunk_skipc (chunk, p); } if (comments_nested != 0) { @@ -157,6 +205,10 @@ ucl_skip_comments (struct ucl_parser *parser) } } + if (beg && p > beg && (parser->flags & UCL_PARSER_SAVE_COMMENTS)) { + ucl_save_comment (parser, beg, p - beg); + } + return true; } @@ -207,7 +259,7 @@ ucl_lex_time_multiplier (const unsigned char c) { {'h', 60 * 60}, {'d', 60 * 60 * 24}, {'w', 60 * 60 * 24 * 7}, - {'y', 60 * 60 * 24 * 7 * 365} + {'y', 60 * 60 * 24 * 365} }; int i; @@ -451,6 +503,11 @@ ucl_expand_variable (struct ucl_parser *parser, unsigned char **dst, size_t out_len = 0; bool vars_found = false; + if (parser->flags & UCL_PARSER_DISABLE_MACRO) { + *dst = NULL; + return in_len; + } + p = src; while (p != end) { if (*p == '$') { @@ -590,12 +647,14 @@ ucl_parser_add_container (ucl_object_t *obj, struct ucl_parser *parser, } st = UCL_ALLOC (sizeof (struct ucl_stack)); + if (st == NULL) { ucl_set_err (parser, UCL_EINTERNAL, "cannot allocate memory for an object", &parser->err); ucl_object_unref (obj); return NULL; } + st->obj = obj; st->level = level; LL_PREPEND (parser->stack, st); @@ -1009,6 +1068,7 @@ ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj { ucl_hash_t *container; ucl_object_t *tobj; + char errmsg[256]; container = parser->stack->obj->value.ov; @@ -1067,31 +1127,43 @@ ucl_parser_process_object_element (struct ucl_parser *parser, ucl_object_t *nobj break; case UCL_DUPLICATE_ERROR: - ucl_create_err (&parser->err, "error while parsing %s: " - "line: %d, column: %d: duplicate element for key '%s' " - "has been found", - parser->cur_file ? parser->cur_file : "", - parser->chunks->line, parser->chunks->column, nobj->key); + snprintf(errmsg, sizeof(errmsg), + "duplicate element for key '%s' found", + nobj->key); + ucl_set_err (parser, UCL_EMERGE, errmsg, &parser->err); return false; case UCL_DUPLICATE_MERGE: /* * Here we do have some old object so we just push it on top of objects stack + * Check priority and then perform the merge on the remaining objects */ if (tobj->type == UCL_OBJECT || tobj->type == UCL_ARRAY) { ucl_object_unref (nobj); nobj = tobj; } - else { - /* For other types we create implicit array as usual */ + else if (priold == prinew) { ucl_parser_append_elt (parser, container, tobj, nobj); } + else if (priold > prinew) { + /* + * We add this new object to a list of trash objects just to ensure + * that it won't come to any real object + * XXX: rather inefficient approach + */ + DL_APPEND (parser->trash_objs, nobj); + } + else { + ucl_hash_replace (container, tobj, nobj); + ucl_object_unref (tobj); + } break; } } parser->stack->obj->value.ov = container; parser->cur_obj = nobj; + ucl_attach_comment (parser, nobj, false); return true; } @@ -1120,7 +1192,10 @@ ucl_parse_key (struct ucl_parser *parser, struct ucl_chunk *chunk, if (*p == '.') { /* It is macro actually */ - ucl_chunk_skipc (chunk, p); + if (!(parser->flags & UCL_PARSER_DISABLE_MACRO)) { + ucl_chunk_skipc (chunk, p); + } + parser->prev_state = parser->state; parser->state = UCL_STATE_MACRO_NAME; *end_of_object = false; @@ -1461,6 +1536,7 @@ ucl_parser_get_container (struct ucl_parser *parser) } parser->cur_obj = obj; + ucl_attach_comment (parser, obj, false); } else { /* Object has been already allocated */ @@ -1511,6 +1587,10 @@ ucl_parse_value (struct ucl_parser *parser, struct ucl_chunk *chunk) } obj = ucl_parser_get_container (parser); + if (!obj) { + return false; + } + str_len = chunk->pos - c - 2; obj->type = UCL_STRING; if ((str_len = ucl_copy_or_store_ptr (parser, c + 1, @@ -1707,12 +1787,19 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) parser->stack = st->next; UCL_FREE (sizeof (struct ucl_stack), st); + if (parser->cur_obj) { + ucl_attach_comment (parser, parser->cur_obj, true); + } + while (parser->stack != NULL) { st = parser->stack; + if (st->next == NULL || st->next->level == st->level) { break; } + parser->stack = st->next; + parser->cur_obj = st->obj; UCL_FREE (sizeof (struct ucl_stack), st); } } @@ -1752,6 +1839,109 @@ ucl_parse_after_value (struct ucl_parser *parser, struct ucl_chunk *chunk) return true; } +static bool +ucl_skip_macro_as_comment (struct ucl_parser *parser, + struct ucl_chunk *chunk) +{ + const unsigned char *p, *c; + enum { + macro_skip_start = 0, + macro_has_symbols, + macro_has_obrace, + macro_has_quote, + macro_has_backslash, + macro_has_sqbrace, + macro_save + } state = macro_skip_start, prev_state = macro_skip_start; + + p = chunk->pos; + c = chunk->pos; + + while (p < chunk->end) { + switch (state) { + case macro_skip_start: + if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE)) { + state = macro_has_symbols; + } + else if (ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { + state = macro_save; + continue; + } + + ucl_chunk_skipc (chunk, p); + break; + + case macro_has_symbols: + if (*p == '{') { + state = macro_has_sqbrace; + } + else if (*p == '(') { + state = macro_has_obrace; + } + else if (*p == '"') { + state = macro_has_quote; + } + else if (*p == '\n') { + state = macro_save; + continue; + } + + ucl_chunk_skipc (chunk, p); + break; + + case macro_has_obrace: + if (*p == '\\') { + prev_state = state; + state = macro_has_backslash; + } + else if (*p == ')') { + state = macro_has_symbols; + } + + ucl_chunk_skipc (chunk, p); + break; + + case macro_has_sqbrace: + if (*p == '\\') { + prev_state = state; + state = macro_has_backslash; + } + else if (*p == '}') { + state = macro_save; + } + + ucl_chunk_skipc (chunk, p); + break; + + case macro_has_quote: + if (*p == '\\') { + prev_state = state; + state = macro_has_backslash; + } + else if (*p == '"') { + state = macro_save; + } + + ucl_chunk_skipc (chunk, p); + break; + + case macro_has_backslash: + state = prev_state; + ucl_chunk_skipc (chunk, p); + break; + + case macro_save: + if (parser->flags & UCL_PARSER_SAVE_COMMENTS) { + ucl_save_comment (parser, c, p - c); + } + + return true; + } + } + + return false; +} + /** * Handle macro data * @param parser @@ -2024,7 +2214,7 @@ ucl_state_machine (struct ucl_parser *parser) while (p < chunk->end && ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE)) { ucl_chunk_skipc (chunk, p); } - if (*p == '}') { + if (p == chunk->end || *p == '}') { /* We have the end of an object */ parser->state = UCL_STATE_AFTER_VALUE; continue; @@ -2067,7 +2257,7 @@ ucl_state_machine (struct ucl_parser *parser) break; case UCL_STATE_VALUE: /* We need to check what we do have */ - if (!ucl_parse_value (parser, chunk)) { + if (!parser->cur_obj || !ucl_parse_value (parser, chunk)) { parser->prev_state = parser->state; parser->state = UCL_STATE_ERROR; return false; @@ -2095,43 +2285,61 @@ ucl_state_machine (struct ucl_parser *parser) /* Skip everything at the end */ return true; } + p = chunk->pos; break; case UCL_STATE_MACRO_NAME: - if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && - *p != '(') { - ucl_chunk_skipc (chunk, p); - } - else { - if (p - c > 0) { - /* We got macro name */ - macro_len = (size_t) (p - c); - HASH_FIND (hh, parser->macroes, c, macro_len, macro); - if (macro == NULL) { - ucl_create_err (&parser->err, - "error on line %d at column %d: " - "unknown macro: '%.*s', character: '%c'", - chunk->line, - chunk->column, - (int) (p - c), - c, - *chunk->pos); - parser->state = UCL_STATE_ERROR; - return false; - } - /* Now we need to skip all spaces */ - SKIP_SPACES_COMMENTS(parser, chunk, p); - parser->state = UCL_STATE_MACRO; - } - else { - /* We have invalid macro name */ + if (parser->flags & UCL_PARSER_DISABLE_MACRO) { + if (!ucl_skip_macro_as_comment (parser, chunk)) { + /* We have invalid macro */ ucl_create_err (&parser->err, - "error on line %d at column %d: invalid macro name", + "error on line %d at column %d: invalid macro", chunk->line, chunk->column); parser->state = UCL_STATE_ERROR; return false; } + else { + p = chunk->pos; + parser->state = parser->prev_state; + } + } + else { + if (!ucl_test_character (*p, UCL_CHARACTER_WHITESPACE_UNSAFE) && + *p != '(') { + ucl_chunk_skipc (chunk, p); + } + else { + if (c != NULL && p - c > 0) { + /* We got macro name */ + macro_len = (size_t) (p - c); + HASH_FIND (hh, parser->macroes, c, macro_len, macro); + if (macro == NULL) { + ucl_create_err (&parser->err, + "error on line %d at column %d: " + "unknown macro: '%.*s', character: '%c'", + chunk->line, + chunk->column, + (int) (p - c), + c, + *chunk->pos); + parser->state = UCL_STATE_ERROR; + return false; + } + /* Now we need to skip all spaces */ + SKIP_SPACES_COMMENTS(parser, chunk, p); + parser->state = UCL_STATE_MACRO; + } + else { + /* We have invalid macro name */ + ucl_create_err (&parser->err, + "error on line %d at column %d: invalid macro name", + chunk->line, + chunk->column); + parser->state = UCL_STATE_ERROR; + return false; + } + } } break; case UCL_STATE_MACRO: @@ -2154,7 +2362,8 @@ ucl_state_machine (struct ucl_parser *parser) macro_len = ucl_expand_variable (parser, ¯o_escaped, macro_start, macro_len); parser->state = parser->prev_state; - if (macro_escaped == NULL) { + + if (macro_escaped == NULL && macro != NULL) { if (macro->is_context) { ret = macro->h.context_handler (macro_start, macro_len, macro_args, @@ -2166,7 +2375,7 @@ ucl_state_machine (struct ucl_parser *parser) macro->ud); } } - else { + else if (macro != NULL) { if (macro->is_context) { ret = macro->h.context_handler (macro_escaped, macro_len, macro_args, @@ -2180,21 +2389,27 @@ ucl_state_machine (struct ucl_parser *parser) UCL_FREE (macro_len + 1, macro_escaped); } + else { + ret = false; + ucl_set_err (parser, UCL_EINTERNAL, + "internal error: parser has macro undefined", &parser->err); + } /* * Chunk can be modified within macro handler */ chunk = parser->chunks; p = chunk->pos; + if (macro_args) { ucl_object_unref (macro_args); } + if (!ret) { return false; } break; default: - /* TODO: add all states */ ucl_set_err (parser, UCL_EINTERNAL, "internal error: parser is in an unknown state", &parser->err); parser->state = UCL_STATE_ERROR; @@ -2202,35 +2417,54 @@ ucl_state_machine (struct ucl_parser *parser) } } + if (parser->last_comment) { + if (parser->cur_obj) { + ucl_attach_comment (parser, parser->cur_obj, true); + } + else if (parser->stack && parser->stack->obj) { + ucl_attach_comment (parser, parser->stack->obj, true); + } + else if (parser->top_obj) { + ucl_attach_comment (parser, parser->top_obj, true); + } + else { + ucl_object_unref (parser->last_comment); + } + } + return true; } struct ucl_parser* ucl_parser_new (int flags) { - struct ucl_parser *new; + struct ucl_parser *parser; - new = UCL_ALLOC (sizeof (struct ucl_parser)); - if (new == NULL) { + parser = UCL_ALLOC (sizeof (struct ucl_parser)); + if (parser == NULL) { return NULL; } - memset (new, 0, sizeof (struct ucl_parser)); + memset (parser, 0, sizeof (struct ucl_parser)); - ucl_parser_register_macro (new, "include", ucl_include_handler, new); - ucl_parser_register_macro (new, "try_include", ucl_try_include_handler, new); - ucl_parser_register_macro (new, "includes", ucl_includes_handler, new); - ucl_parser_register_macro (new, "priority", ucl_priority_handler, new); - ucl_parser_register_macro (new, "load", ucl_load_handler, new); - ucl_parser_register_context_macro (new, "inherit", ucl_inherit_handler, new); + ucl_parser_register_macro (parser, "include", ucl_include_handler, parser); + ucl_parser_register_macro (parser, "try_include", ucl_try_include_handler, parser); + ucl_parser_register_macro (parser, "includes", ucl_includes_handler, parser); + ucl_parser_register_macro (parser, "priority", ucl_priority_handler, parser); + ucl_parser_register_macro (parser, "load", ucl_load_handler, parser); + ucl_parser_register_context_macro (parser, "inherit", ucl_inherit_handler, parser); - new->flags = flags; - new->includepaths = NULL; + parser->flags = flags; + parser->includepaths = NULL; + + if (flags & UCL_PARSER_SAVE_COMMENTS) { + parser->comments = ucl_object_typed_new (UCL_OBJECT); + } /* Initial assumption about filevars */ - ucl_parser_set_filevars (new, NULL, false); + ucl_parser_set_filevars (parser, NULL, false); - return new; + return parser; } bool @@ -2363,14 +2597,16 @@ ucl_parser_add_chunk_full (struct ucl_parser *parser, const unsigned char *data, return false; } - if (data == NULL) { - ucl_create_err (&parser->err, "invalid chunk added"); - return false; - } if (len == 0) { parser->top_obj = ucl_object_new_full (UCL_OBJECT, priority); return true; } + + if (data == NULL) { + ucl_create_err (&parser->err, "invalid chunk added"); + return false; + } + if (parser->state != UCL_STATE_ERROR) { chunk = UCL_ALLOC (sizeof (struct ucl_chunk)); if (chunk == NULL) { diff --git a/contrib/libucl/src/ucl_schema.c b/contrib/libucl/src/ucl_schema.c index 834b62accb5f..4d7c871ff9f8 100644 --- a/contrib/libucl/src/ucl_schema.c +++ b/contrib/libucl/src/ucl_schema.c @@ -43,72 +43,8 @@ static bool ucl_schema_validate (const ucl_object_t *schema, const ucl_object_t *obj, bool try_array, struct ucl_schema_error *err, - const ucl_object_t *root); - -static bool -ucl_string_to_type (const char *input, ucl_type_t *res) -{ - if (strcasecmp (input, "object") == 0) { - *res = UCL_OBJECT; - } - else if (strcasecmp (input, "array") == 0) { - *res = UCL_ARRAY; - } - else if (strcasecmp (input, "integer") == 0) { - *res = UCL_INT; - } - else if (strcasecmp (input, "number") == 0) { - *res = UCL_FLOAT; - } - else if (strcasecmp (input, "string") == 0) { - *res = UCL_STRING; - } - else if (strcasecmp (input, "boolean") == 0) { - *res = UCL_BOOLEAN; - } - else if (strcasecmp (input, "null") == 0) { - *res = UCL_NULL; - } - else { - return false; - } - - return true; -} - -static const char * -ucl_object_type_to_string (ucl_type_t type) -{ - const char *res = "unknown"; - - switch (type) { - case UCL_OBJECT: - res = "object"; - break; - case UCL_ARRAY: - res = "array"; - break; - case UCL_INT: - res = "integer"; - break; - case UCL_FLOAT: - case UCL_TIME: - res = "number"; - break; - case UCL_STRING: - res = "string"; - break; - case UCL_BOOLEAN: - res = "boolean"; - break; - case UCL_NULL: - case UCL_USERDATA: - res = "null"; - break; - } - - return res; -} + const ucl_object_t *root, + ucl_object_t *ext_ref); /* * Create validation error @@ -142,7 +78,7 @@ ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) ucl_object_iter_t iter = NULL; if (regcomp (®, pattern, REG_EXTENDED | REG_NOSUB) == 0) { - while ((elt = ucl_iterate_object (obj, &iter, true)) != NULL) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { if (regexec (®, ucl_object_key (elt), 0, NULL, 0) == 0) { res = elt; break; @@ -160,20 +96,21 @@ ucl_schema_test_pattern (const ucl_object_t *obj, const char *pattern) static bool ucl_schema_validate_dependencies (const ucl_object_t *deps, const ucl_object_t *obj, struct ucl_schema_error *err, - const ucl_object_t *root) + const ucl_object_t *root, + ucl_object_t *ext_ref) { const ucl_object_t *elt, *cur, *cur_dep; ucl_object_iter_t iter = NULL, piter; bool ret = true; - while (ret && (cur = ucl_iterate_object (deps, &iter, true)) != NULL) { - elt = ucl_object_find_key (obj, ucl_object_key (cur)); + while (ret && (cur = ucl_object_iterate (deps, &iter, true)) != NULL) { + elt = ucl_object_lookup (obj, ucl_object_key (cur)); if (elt != NULL) { /* Need to check dependencies */ if (cur->type == UCL_ARRAY) { piter = NULL; - while (ret && (cur_dep = ucl_iterate_object (cur, &piter, true)) != NULL) { - if (ucl_object_find_key (obj, ucl_object_tostring (cur_dep)) == NULL) { + while (ret && (cur_dep = ucl_object_iterate (cur, &piter, true)) != NULL) { + if (ucl_object_lookup (obj, ucl_object_tostring (cur_dep)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_DEPENDENCY, elt, "dependency %s is missing for key %s", ucl_object_tostring (cur_dep), ucl_object_key (cur)); @@ -183,7 +120,7 @@ ucl_schema_validate_dependencies (const ucl_object_t *deps, } } else if (cur->type == UCL_OBJECT) { - ret = ucl_schema_validate (cur, obj, true, err, root); + ret = ucl_schema_validate (cur, obj, true, err, root, ext_ref); } } } @@ -197,7 +134,8 @@ ucl_schema_validate_dependencies (const ucl_object_t *deps, static bool ucl_schema_validate_object (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err, - const ucl_object_t *root) + const ucl_object_t *root, + ucl_object_t *ext_ref) { const ucl_object_t *elt, *prop, *found, *additional_schema = NULL, *required = NULL, *pat, *pelt; @@ -205,14 +143,15 @@ ucl_schema_validate_object (const ucl_object_t *schema, bool ret = true, allow_additional = true; int64_t minmax; - while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) { + while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "properties") == 0) { piter = NULL; - while (ret && (prop = ucl_iterate_object (elt, &piter, true)) != NULL) { - found = ucl_object_find_key (obj, ucl_object_key (prop)); + while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { + found = ucl_object_lookup (obj, ucl_object_key (prop)); if (found) { - ret = ucl_schema_validate (prop, found, true, err, root); + ret = ucl_schema_validate (prop, found, true, err, root, + ext_ref); } } } @@ -267,16 +206,18 @@ ucl_schema_validate_object (const ucl_object_t *schema, } else if (strcmp (ucl_object_key (elt), "patternProperties") == 0) { piter = NULL; - while (ret && (prop = ucl_iterate_object (elt, &piter, true)) != NULL) { + while (ret && (prop = ucl_object_iterate (elt, &piter, true)) != NULL) { found = ucl_schema_test_pattern (obj, ucl_object_key (prop)); if (found) { - ret = ucl_schema_validate (prop, found, true, err, root); + ret = ucl_schema_validate (prop, found, true, err, root, + ext_ref); } } } else if (elt->type == UCL_OBJECT && strcmp (ucl_object_key (elt), "dependencies") == 0) { - ret = ucl_schema_validate_dependencies (elt, obj, err, root); + ret = ucl_schema_validate_dependencies (elt, obj, err, root, + ext_ref); } } @@ -285,14 +226,14 @@ ucl_schema_validate_object (const ucl_object_t *schema, if (!allow_additional || additional_schema != NULL) { /* Check if we have exactly the same properties in schema and object */ iter = NULL; - prop = ucl_object_find_key (schema, "properties"); - while ((elt = ucl_iterate_object (obj, &iter, true)) != NULL) { - found = ucl_object_find_key (prop, ucl_object_key (elt)); + prop = ucl_object_lookup (schema, "properties"); + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { + found = ucl_object_lookup (prop, ucl_object_key (elt)); if (found == NULL) { /* Try patternProperties */ piter = NULL; - pat = ucl_object_find_key (schema, "patternProperties"); - while ((pelt = ucl_iterate_object (pat, &piter, true)) != NULL) { + pat = ucl_object_lookup (schema, "patternProperties"); + while ((pelt = ucl_object_iterate (pat, &piter, true)) != NULL) { found = ucl_schema_test_pattern (obj, ucl_object_key (pelt)); if (found != NULL) { break; @@ -308,7 +249,8 @@ ucl_schema_validate_object (const ucl_object_t *schema, break; } else if (additional_schema != NULL) { - if (!ucl_schema_validate (additional_schema, elt, true, err, root)) { + if (!ucl_schema_validate (additional_schema, elt, + true, err, root, ext_ref)) { ret = false; break; } @@ -319,8 +261,8 @@ ucl_schema_validate_object (const ucl_object_t *schema, /* Required properties */ if (required != NULL) { iter = NULL; - while ((elt = ucl_iterate_object (required, &iter, true)) != NULL) { - if (ucl_object_find_key (obj, ucl_object_tostring (elt)) == NULL) { + while ((elt = ucl_object_iterate (required, &iter, true)) != NULL) { + if (ucl_object_lookup (obj, ucl_object_tostring (elt)) == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_MISSING_PROPERTY, obj, "object has missing property %s", ucl_object_tostring (elt)); @@ -345,7 +287,7 @@ ucl_schema_validate_number (const ucl_object_t *schema, double constraint, val; const double alpha = 1e-16; - while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) { + while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "multipleOf") == 0) { constraint = ucl_object_todouble (elt); @@ -367,7 +309,7 @@ ucl_schema_validate_number (const ucl_object_t *schema, else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "maximum") == 0) { constraint = ucl_object_todouble (elt); - test = ucl_object_find_key (schema, "exclusiveMaximum"); + test = ucl_object_lookup (schema, "exclusiveMaximum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } @@ -383,7 +325,7 @@ ucl_schema_validate_number (const ucl_object_t *schema, else if ((elt->type == UCL_FLOAT || elt->type == UCL_INT) && strcmp (ucl_object_key (elt), "minimum") == 0) { constraint = ucl_object_todouble (elt); - test = ucl_object_find_key (schema, "exclusiveMinimum"); + test = ucl_object_lookup (schema, "exclusiveMinimum"); if (test && test->type == UCL_BOOLEAN) { exclusive = ucl_object_toboolean (test); } @@ -413,7 +355,7 @@ ucl_schema_validate_string (const ucl_object_t *schema, regex_t re; #endif - while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) { + while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (elt->type == UCL_INT && strcmp (ucl_object_key (elt), "maxLength") == 0) { constraint = ucl_object_toint (elt); @@ -487,7 +429,7 @@ ucl_schema_array_is_unique (const ucl_object_t *obj, struct ucl_schema_error *er struct ucl_compare_node *node, test, *nodes = NULL, *tmp; bool ret = true; - while ((elt = ucl_iterate_object (obj, &iter, true)) != NULL) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { test.obj = elt; node = TREE_FIND (&tree, ucl_compare_node, link, &test); if (node != NULL) { @@ -518,7 +460,8 @@ ucl_schema_array_is_unique (const ucl_object_t *obj, struct ucl_schema_error *er static bool ucl_schema_validate_array (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err, - const ucl_object_t *root) + const ucl_object_t *root, + ucl_object_t *ext_ref) { const ucl_object_t *elt, *it, *found, *additional_schema = NULL, *first_unvalidated = NULL; @@ -527,13 +470,14 @@ ucl_schema_validate_array (const ucl_object_t *schema, int64_t minmax; unsigned int idx = 0; - while (ret && (elt = ucl_iterate_object (schema, &iter, true)) != NULL) { + while (ret && (elt = ucl_object_iterate (schema, &iter, true)) != NULL) { if (strcmp (ucl_object_key (elt), "items") == 0) { if (elt->type == UCL_ARRAY) { found = ucl_array_head (obj); - while (ret && (it = ucl_iterate_object (elt, &piter, true)) != NULL) { + while (ret && (it = ucl_object_iterate (elt, &piter, true)) != NULL) { if (found) { - ret = ucl_schema_validate (it, found, false, err, root); + ret = ucl_schema_validate (it, found, false, err, + root, ext_ref); found = ucl_array_find_index (obj, ++idx); } } @@ -544,8 +488,9 @@ ucl_schema_validate_array (const ucl_object_t *schema, } else if (elt->type == UCL_OBJECT) { /* Validate all items using the specified schema */ - while (ret && (it = ucl_iterate_object (obj, &piter, true)) != NULL) { - ret = ucl_schema_validate (elt, it, false, err, root); + while (ret && (it = ucl_object_iterate (obj, &piter, true)) != NULL) { + ret = ucl_schema_validate (elt, it, false, err, root, + ext_ref); } } else { @@ -612,7 +557,7 @@ ucl_schema_validate_array (const ucl_object_t *schema, elt = ucl_array_find_index (obj, idx); while (elt) { if (!ucl_schema_validate (additional_schema, elt, false, - err, root)) { + err, root, ext_ref)) { ret = false; break; } @@ -649,7 +594,7 @@ ucl_schema_type_is_allowed (const ucl_object_t *type, const ucl_object_t *obj, if (type->type == UCL_ARRAY) { /* One of allowed types */ - while ((elt = ucl_iterate_object (type, &iter, true)) != NULL) { + while ((elt = ucl_object_iterate (type, &iter, true)) != NULL) { if (ucl_schema_type_is_allowed (elt, obj, err)) { return true; } @@ -657,7 +602,7 @@ ucl_schema_type_is_allowed (const ucl_object_t *type, const ucl_object_t *obj, } else if (type->type == UCL_STRING) { type_str = ucl_object_tostring (type); - if (!ucl_string_to_type (type_str, &t)) { + if (!ucl_object_string_to_type (type_str, &t)) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, type, "Type attribute is invalid in schema"); return false; @@ -697,7 +642,7 @@ ucl_schema_validate_enum (const ucl_object_t *en, const ucl_object_t *obj, const ucl_object_t *elt; bool ret = false; - while ((elt = ucl_iterate_object (en, &iter, true)) != NULL) { + while ((elt = ucl_object_iterate (en, &iter, true)) != NULL) { if (ucl_object_compare (elt, obj) == 0) { ret = true; break; @@ -727,7 +672,7 @@ ucl_schema_resolve_ref_component (const ucl_object_t *cur, if (cur->type == UCL_OBJECT) { /* Find a key inside an object */ - res = ucl_object_find_keyl (cur, refc, len); + res = ucl_object_lookup_len (cur, refc, len); if (res == NULL) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, cur, "reference %s is invalid, missing path component", refc); @@ -771,31 +716,114 @@ ucl_schema_resolve_ref_component (const ucl_object_t *cur, */ static const ucl_object_t * ucl_schema_resolve_ref (const ucl_object_t *root, const char *ref, - struct ucl_schema_error *err) + struct ucl_schema_error *err, ucl_object_t *ext_ref, + ucl_object_t const ** nroot) { - const char *p, *c; - const ucl_object_t *res = NULL; - + UT_string *url_err = NULL; + struct ucl_parser *parser; + const ucl_object_t *res = NULL, *ext_obj = NULL; + ucl_object_t *url_obj; + const char *p, *c, *hash_ptr = NULL; + char *url_copy = NULL; + unsigned char *url_buf; + size_t url_buflen; if (ref[0] != '#') { - ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, - "reference %s is invalid, not started with #", ref); - return NULL; - } - if (ref[1] == '/') { - p = &ref[2]; - } - else if (ref[1] == '\0') { - return root; + hash_ptr = strrchr (ref, '#'); + + if (hash_ptr) { + url_copy = malloc (hash_ptr - ref + 1); + + if (url_copy == NULL) { + ucl_schema_create_error (err, UCL_SCHEMA_INTERNAL_ERROR, root, + "cannot allocate memory"); + return NULL; + } + + ucl_strlcpy (url_copy, ref, hash_ptr - ref + 1); + p = url_copy; + } + else { + /* Full URL */ + p = ref; + } + + ext_obj = ucl_object_lookup (ext_ref, p); + + if (ext_obj == NULL) { + if (ucl_strnstr (p, "://", strlen (p)) != NULL) { + if (!ucl_fetch_url (p, &url_buf, &url_buflen, &url_err, true)) { + + ucl_schema_create_error (err, + UCL_SCHEMA_INVALID_SCHEMA, + root, + "cannot fetch reference %s: %s", + p, + url_err != NULL ? utstring_body (url_err) + : "unknown"); + free (url_copy); + + return NULL; + } + } + else { + if (!ucl_fetch_file (p, &url_buf, &url_buflen, &url_err, + true)) { + ucl_schema_create_error (err, + UCL_SCHEMA_INVALID_SCHEMA, + root, + "cannot fetch reference %s: %s", + p, + url_err != NULL ? utstring_body (url_err) + : "unknown"); + free (url_copy); + + return NULL; + } + } + + parser = ucl_parser_new (0); + + if (!ucl_parser_add_chunk (parser, url_buf, url_buflen)) { + ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, + "cannot fetch reference %s: %s", p, + ucl_parser_get_error (parser)); + ucl_parser_free (parser); + free (url_copy); + + return NULL; + } + + url_obj = ucl_parser_get_object (parser); + ext_obj = url_obj; + ucl_object_insert_key (ext_ref, url_obj, p, 0, true); + free (url_buf); + } + + free (url_copy); + + if (hash_ptr) { + p = hash_ptr + 1; + } + else { + p = ""; + } } else { - ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, root, - "reference %s is invalid, not started with #/", ref); - return NULL; + p = ref + 1; + } + + res = ext_obj != NULL ? ext_obj : root; + *nroot = res; + + if (*p == '/') { + p++; + } + else if (*p == '\0') { + return res; } c = p; - res = root; while (*p != '\0') { if (*p == '/') { @@ -835,7 +863,7 @@ ucl_schema_validate_values (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *elt, *cur; int64_t constraint, i; - elt = ucl_object_find_key (schema, "maxValues"); + elt = ucl_object_lookup (schema, "maxValues"); if (elt != NULL && elt->type == UCL_INT) { constraint = ucl_object_toint (elt); cur = obj; @@ -851,7 +879,7 @@ ucl_schema_validate_values (const ucl_object_t *schema, const ucl_object_t *obj, cur = cur->next; } } - elt = ucl_object_find_key (schema, "minValues"); + elt = ucl_object_lookup (schema, "minValues"); if (elt != NULL && elt->type == UCL_INT) { constraint = ucl_object_toint (elt); cur = obj; @@ -878,15 +906,17 @@ static bool ucl_schema_validate (const ucl_object_t *schema, const ucl_object_t *obj, bool try_array, struct ucl_schema_error *err, - const ucl_object_t *root) + const ucl_object_t *root, + ucl_object_t *external_refs) { - const ucl_object_t *elt, *cur; + const ucl_object_t *elt, *cur, *ref_root; ucl_object_iter_t iter = NULL; bool ret; if (schema->type != UCL_OBJECT) { ucl_schema_create_error (err, UCL_SCHEMA_INVALID_SCHEMA, schema, - "schema is %s instead of object", ucl_object_type_to_string (schema->type)); + "schema is %s instead of object", + ucl_object_type_to_string (schema->type)); return false; } @@ -898,36 +928,36 @@ ucl_schema_validate (const ucl_object_t *schema, return false; } LL_FOREACH (obj, cur) { - if (!ucl_schema_validate (schema, cur, false, err, root)) { + if (!ucl_schema_validate (schema, cur, false, err, root, external_refs)) { return false; } } return true; } - elt = ucl_object_find_key (schema, "enum"); + elt = ucl_object_lookup (schema, "enum"); if (elt != NULL && elt->type == UCL_ARRAY) { if (!ucl_schema_validate_enum (elt, obj, err)) { return false; } } - elt = ucl_object_find_key (schema, "allOf"); + elt = ucl_object_lookup (schema, "allOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; - while ((cur = ucl_iterate_object (elt, &iter, true)) != NULL) { - ret = ucl_schema_validate (cur, obj, true, err, root); + while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { + ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); if (!ret) { return false; } } } - elt = ucl_object_find_key (schema, "anyOf"); + elt = ucl_object_lookup (schema, "anyOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; - while ((cur = ucl_iterate_object (elt, &iter, true)) != NULL) { - ret = ucl_schema_validate (cur, obj, true, err, root); + while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { + ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); if (ret) { break; } @@ -941,15 +971,15 @@ ucl_schema_validate (const ucl_object_t *schema, } } - elt = ucl_object_find_key (schema, "oneOf"); + elt = ucl_object_lookup (schema, "oneOf"); if (elt != NULL && elt->type == UCL_ARRAY) { iter = NULL; ret = false; - while ((cur = ucl_iterate_object (elt, &iter, true)) != NULL) { + while ((cur = ucl_object_iterate (elt, &iter, true)) != NULL) { if (!ret) { - ret = ucl_schema_validate (cur, obj, true, err, root); + ret = ucl_schema_validate (cur, obj, true, err, root, external_refs); } - else if (ucl_schema_validate (cur, obj, true, err, root)) { + else if (ucl_schema_validate (cur, obj, true, err, root, external_refs)) { ret = false; break; } @@ -959,9 +989,9 @@ ucl_schema_validate (const ucl_object_t *schema, } } - elt = ucl_object_find_key (schema, "not"); + elt = ucl_object_lookup (schema, "not"); if (elt != NULL && elt->type == UCL_OBJECT) { - if (ucl_schema_validate (elt, obj, true, err, root)) { + if (ucl_schema_validate (elt, obj, true, err, root, external_refs)) { return false; } else { @@ -970,28 +1000,32 @@ ucl_schema_validate (const ucl_object_t *schema, } } - elt = ucl_object_find_key (schema, "$ref"); + elt = ucl_object_lookup (schema, "$ref"); if (elt != NULL) { - cur = ucl_schema_resolve_ref (root, ucl_object_tostring (elt), err); + ref_root = root; + cur = ucl_schema_resolve_ref (root, ucl_object_tostring (elt), + err, external_refs, &ref_root); + if (cur == NULL) { return false; } - if (!ucl_schema_validate (cur, obj, try_array, err, root)) { + if (!ucl_schema_validate (cur, obj, try_array, err, ref_root, + external_refs)) { return false; } } - elt = ucl_object_find_key (schema, "type"); + elt = ucl_object_lookup (schema, "type"); if (!ucl_schema_type_is_allowed (elt, obj, err)) { return false; } switch (obj->type) { case UCL_OBJECT: - return ucl_schema_validate_object (schema, obj, err, root); + return ucl_schema_validate_object (schema, obj, err, root, external_refs); break; case UCL_ARRAY: - return ucl_schema_validate_array (schema, obj, err, root); + return ucl_schema_validate_array (schema, obj, err, root, external_refs); break; case UCL_INT: case UCL_FLOAT: @@ -1011,5 +1045,37 @@ bool ucl_object_validate (const ucl_object_t *schema, const ucl_object_t *obj, struct ucl_schema_error *err) { - return ucl_schema_validate (schema, obj, true, err, schema); + return ucl_object_validate_root_ext (schema, obj, schema, NULL, err); +} + +bool +ucl_object_validate_root (const ucl_object_t *schema, + const ucl_object_t *obj, + const ucl_object_t *root, + struct ucl_schema_error *err) +{ + return ucl_object_validate_root_ext (schema, obj, root, NULL, err); +} + +bool +ucl_object_validate_root_ext (const ucl_object_t *schema, + const ucl_object_t *obj, + const ucl_object_t *root, + ucl_object_t *ext_refs, + struct ucl_schema_error *err) +{ + bool ret, need_unref = false; + + if (ext_refs == NULL) { + ext_refs = ucl_object_typed_new (UCL_OBJECT); + need_unref = true; + } + + ret = ucl_schema_validate (schema, obj, true, err, root, ext_refs); + + if (need_unref) { + ucl_object_unref (ext_refs); + } + + return ret; } diff --git a/contrib/libucl/src/ucl_sexp.c b/contrib/libucl/src/ucl_sexp.c index 3ca7eb102925..1ad93d23458e 100644 --- a/contrib/libucl/src/ucl_sexp.c +++ b/contrib/libucl/src/ucl_sexp.c @@ -108,6 +108,7 @@ ucl_parse_csexp (struct ucl_parser *parser) if (st->obj == NULL) { ucl_create_err (&parser->err, "no memory"); state = parse_err; + free (st); continue; } @@ -205,6 +206,7 @@ ucl_parse_csexp (struct ucl_parser *parser) } free (st); + st = NULL; p++; NEXT_STATE; break; @@ -221,4 +223,4 @@ ucl_parse_csexp (struct ucl_parser *parser) } return true; -} \ No newline at end of file +} diff --git a/contrib/libucl/src/ucl_util.c b/contrib/libucl/src/ucl_util.c index 730a5c4afd57..261bf95f5b7d 100644 --- a/contrib/libucl/src/ucl_util.c +++ b/contrib/libucl/src/ucl_util.c @@ -27,6 +27,7 @@ #include "ucl_chartable.h" #include "kvec.h" #include +#include /* for snprintf */ #ifndef _WIN32 #include @@ -50,6 +51,8 @@ typedef kvec_t(ucl_object_t *) ucl_array_t; #endif #ifdef CURL_FOUND +/* Seems to be broken */ +#define CURL_DISABLE_TYPECHECK 1 #include #endif #ifdef HAVE_FETCH_H @@ -236,7 +239,7 @@ ucl_object_free_internal (ucl_object_t *obj, bool allow_rec, ucl_object_dtor dto } else if (obj->type == UCL_OBJECT) { if (obj->value.ov != NULL) { - ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func *)dtor); + ucl_hash_destroy (obj->value.ov, (ucl_hash_free_func)dtor); } obj->value.ov = NULL; } @@ -306,6 +309,9 @@ ucl_unescape_json_string (char *str, size_t len) case 'u': /* Unicode escape */ uval = 0; + h ++; /* u character */ + len --; + if (len > 3) { for (i = 0; i < 4; i++) { uval <<= 4; @@ -322,8 +328,7 @@ ucl_unescape_json_string (char *str, size_t len) break; } } - h += 3; - len -= 3; + /* Encode */ if(uval < 0x80) { t[0] = (char)uval; @@ -340,6 +345,8 @@ ucl_unescape_json_string (char *str, size_t len) t[2] = 0x80 + ((uval & 0x003F)); t += 3; } +#if 0 + /* It's not actually supported now */ else if(uval <= 0x10FFFF) { t[0] = 0xF0 + ((uval & 0x1C0000) >> 18); t[1] = 0x80 + ((uval & 0x03F000) >> 12); @@ -347,9 +354,19 @@ ucl_unescape_json_string (char *str, size_t len) t[3] = 0x80 + ((uval & 0x00003F)); t += 4; } +#endif else { *t++ = '?'; } + + /* Consume 4 characters of source */ + h += 4; + len -= 4; + + if (len > 0) { + len --; /* for '\' character */ + } + continue; } else { *t++ = 'u'; @@ -437,7 +454,7 @@ ucl_copy_value_trash (const ucl_object_t *obj) } deconst->flags |= UCL_OBJECT_ALLOCATED_VALUE; } - + return obj->trash_stack[UCL_TRASH_VALUE]; } @@ -504,6 +521,10 @@ ucl_parser_free (struct ucl_parser *parser) free (parser->cur_file); } + if (parser->comments) { + ucl_object_unref (parser->comments); + } + UCL_FREE (sizeof (struct ucl_parser), parser); } @@ -628,7 +649,7 @@ ucl_curl_write_callback (void* contents, size_t size, size_t nmemb, void* ud) * @param buflen target length * @return */ -static bool +bool ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen, UT_string **err, bool must_exist) { @@ -690,8 +711,8 @@ ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen, return false; } curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, ucl_curl_write_callback); - cbdata.buf = *buf; - cbdata.buflen = *buflen; + cbdata.buf = NULL; + cbdata.buflen = 0; curl_easy_setopt (curl, CURLOPT_WRITEDATA, &cbdata); if ((r = curl_easy_perform (curl)) != CURLE_OK) { @@ -723,7 +744,7 @@ ucl_fetch_url (const unsigned char *url, unsigned char **buf, size_t *buflen, * @param buflen target length * @return */ -static bool +bool ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *buflen, UT_string **err, bool must_exist) { @@ -739,7 +760,7 @@ ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *bufl } if (st.st_size == 0) { /* Do not map empty files */ - *buf = ""; + *buf = NULL; *buflen = 0; } else { @@ -752,6 +773,8 @@ ucl_fetch_file (const unsigned char *filename, unsigned char **buf, size_t *bufl close (fd); ucl_create_err (err, "cannot mmap file %s: %s", filename, strerror (errno)); + *buf = NULL; + return false; } *buflen = st.st_size; @@ -848,7 +871,7 @@ ucl_include_url (const unsigned char *data, size_t len, snprintf (urlbuf, sizeof (urlbuf), "%.*s", (int)len, data); if (!ucl_fetch_url (urlbuf, &buf, &buflen, &parser->err, params->must_exist)) { - return (!params->must_exist || false); + return !params->must_exist; } if (params->check_signature) { @@ -968,12 +991,12 @@ ucl_include_file_single (const unsigned char *data, size_t len, ucl_create_err (&parser->err, "cannot verify file %s: %s", filebuf, ERR_error_string (ERR_get_error (), NULL)); - if (siglen > 0) { + if (sigbuf) { ucl_munmap (sigbuf, siglen); } return false; } - if (siglen > 0) { + if (sigbuf) { ucl_munmap (sigbuf, siglen); } #endif @@ -1037,6 +1060,16 @@ ucl_include_file_single (const unsigned char *data, size_t len, else if (old_obj == NULL) { /* Create an object with key: prefix */ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); + + if (nest_obj == NULL) { + ucl_create_err (&parser->err, "cannot allocate memory for an object"); + if (buf) { + ucl_munmap (buf, buflen); + } + + return false; + } + nest_obj->key = params->prefix; nest_obj->keylen = strlen (params->prefix); ucl_copy_key_trash(nest_obj); @@ -1052,6 +1085,14 @@ ucl_include_file_single (const unsigned char *data, size_t len, if (ucl_object_type(old_obj) == UCL_ARRAY) { /* Append to the existing array */ nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); + if (nest_obj == NULL) { + ucl_create_err (&parser->err, "cannot allocate memory for an object"); + if (buf) { + ucl_munmap (buf, buflen); + } + + return false; + } nest_obj->prev = nest_obj; nest_obj->next = NULL; @@ -1060,6 +1101,14 @@ ucl_include_file_single (const unsigned char *data, size_t len, else { /* Convert the object to an array */ new_obj = ucl_object_typed_new (UCL_ARRAY); + if (new_obj == NULL) { + ucl_create_err (&parser->err, "cannot allocate memory for an object"); + if (buf) { + ucl_munmap (buf, buflen); + } + + return false; + } new_obj->key = old_obj->key; new_obj->keylen = old_obj->keylen; new_obj->flags |= UCL_OBJECT_MULTIVALUE; @@ -1067,6 +1116,14 @@ ucl_include_file_single (const unsigned char *data, size_t len, new_obj->next = NULL; nest_obj = ucl_object_new_full (UCL_OBJECT, params->priority); + if (nest_obj == NULL) { + ucl_create_err (&parser->err, "cannot allocate memory for an object"); + if (buf) { + ucl_munmap (buf, buflen); + } + + return false; + } nest_obj->prev = nest_obj; nest_obj->next = NULL; @@ -1085,6 +1142,10 @@ ucl_include_file_single (const unsigned char *data, size_t len, ucl_create_err (&parser->err, "Conflicting type for key: %s", params->prefix); + if (buf) { + ucl_munmap (buf, buflen); + } + return false; } } @@ -1092,18 +1153,21 @@ ucl_include_file_single (const unsigned char *data, size_t len, /* Put all of the content of the include inside that object */ parser->stack->obj->value.ov = container; - if (nest_obj != NULL) { - st = UCL_ALLOC (sizeof (struct ucl_stack)); - if (st == NULL) { - ucl_create_err (&parser->err, "cannot allocate memory for an object"); - ucl_object_unref (nest_obj); - return NULL; + st = UCL_ALLOC (sizeof (struct ucl_stack)); + if (st == NULL) { + ucl_create_err (&parser->err, "cannot allocate memory for an object"); + ucl_object_unref (nest_obj); + + if (buf) { + ucl_munmap (buf, buflen); } - st->obj = nest_obj; - st->level = parser->stack->level; - LL_PREPEND (parser->stack, st); - parser->cur_obj = nest_obj; + + return false; } + st->obj = nest_obj; + st->level = parser->stack->level; + LL_PREPEND (parser->stack, st); + parser->cur_obj = nest_obj; } res = ucl_parser_add_chunk_full (parser, buf, buflen, params->priority, @@ -1232,7 +1296,7 @@ ucl_include_file (const unsigned char *data, size_t len, treat allow_glob/need_glob as a NOOP and just return */ return ucl_include_file_single (data, len, parser, params); #endif - + return true; } @@ -1252,7 +1316,7 @@ ucl_include_common (const unsigned char *data, size_t len, bool default_try, bool default_sign) { - bool allow_url, search; + bool allow_url = false, search = false; const char *duplicate; const ucl_object_t *param; ucl_object_iter_t it = NULL, ip = NULL; @@ -1271,11 +1335,9 @@ ucl_include_common (const unsigned char *data, size_t len, params.strat = UCL_DUPLICATE_APPEND; params.must_exist = !default_try; - search = false; - /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { - while ((param = ucl_iterate_object (args, &it, true)) != NULL) { + while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_BOOLEAN) { if (strncmp (param->key, "try", param->keylen) == 0) { params.must_exist = !ucl_object_toboolean (param); @@ -1450,7 +1512,7 @@ ucl_priority_handler (const unsigned char *data, size_t len, /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { - while ((param = ucl_iterate_object (args, &it, true)) != NULL) { + while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_INT) { if (strncmp (param->key, "priority", param->keylen) == 0) { priority = ucl_object_toint (param); @@ -1506,7 +1568,7 @@ ucl_load_handler (const unsigned char *data, size_t len, size_t buflen; unsigned priority; int64_t iv; - ucl_hash_t *container = NULL; + ucl_object_t *container = NULL; enum ucl_string_flags flags; /* Default values */ @@ -1529,7 +1591,7 @@ ucl_load_handler (const unsigned char *data, size_t len, /* Process arguments */ if (args != NULL && args->type == UCL_OBJECT) { - while ((param = ucl_iterate_object (args, &it, true)) != NULL) { + while ((param = ucl_object_iterate (args, &it, true)) != NULL) { if (param->type == UCL_BOOLEAN) { if (strncmp (param->key, "try", param->keylen) == 0) { try_load = ucl_object_toboolean (param); @@ -1566,21 +1628,39 @@ ucl_load_handler (const unsigned char *data, size_t len, } } - if (prefix == NULL || strlen(prefix) == 0) { + if (prefix == NULL || strlen (prefix) == 0) { ucl_create_err (&parser->err, "No Key specified in load macro"); return false; } if (len > 0) { - asprintf (&load_file, "%.*s", (int)len, data); - if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err, !try_load)) { + load_file = malloc (len + 1); + if (!load_file) { + ucl_create_err (&parser->err, "cannot allocate memory for suffix"); + + return false; + } + + snprintf (load_file, len + 1, "%.*s", (int)len, data); + + if (!ucl_fetch_file (load_file, &buf, &buflen, &parser->err, + !try_load)) { + free (load_file); + return (try_load || false); } - container = parser->stack->obj->value.ov; - old_obj = __DECONST (ucl_object_t *, ucl_hash_search (container, prefix, strlen (prefix))); + free (load_file); + container = parser->stack->obj; + old_obj = __DECONST (ucl_object_t *, ucl_object_lookup (container, + prefix)); + if (old_obj != NULL) { ucl_create_err (&parser->err, "Key %s already exists", prefix); + if (buf) { + ucl_munmap (buf, buflen); + } + return false; } @@ -1592,26 +1672,37 @@ ucl_load_handler (const unsigned char *data, size_t len, } } else if (strcasecmp (target, "int") == 0) { - asprintf(&tmp, "%.*s", (int)buflen, buf); - iv = strtoll(tmp, NULL, 10); - obj = ucl_object_fromint(iv); + tmp = malloc (buflen + 1); + + if (tmp == NULL) { + ucl_create_err (&parser->err, "Memory allocation failed"); + if (buf) { + ucl_munmap (buf, buflen); + } + + return false; + } + + snprintf (tmp, buflen + 1, "%.*s", (int)buflen, buf); + iv = strtoll (tmp, NULL, 10); + obj = ucl_object_fromint (iv); + free (tmp); } - if (buflen > 0) { + if (buf) { ucl_munmap (buf, buflen); } if (obj != NULL) { obj->key = prefix; obj->keylen = strlen (prefix); - ucl_copy_key_trash(obj); + ucl_copy_key_trash (obj); obj->prev = obj; obj->next = NULL; ucl_object_set_priority (obj, priority); - container = ucl_hash_insert_object (container, obj, - parser->flags & UCL_PARSER_KEY_LOWERCASE); - parser->stack->obj->value.ov = container; + ucl_object_insert_key (container, obj, obj->key, obj->keylen, false); } + return true; } @@ -1629,7 +1720,7 @@ ucl_inherit_handler (const unsigned char *data, size_t len, bool replace = false; struct ucl_parser *parser = ud; - parent = ucl_object_find_keyl (ctx, data, len); + parent = ucl_object_lookup_len (ctx, data, len); /* Some sanity checks */ if (parent == NULL || ucl_object_type (parent) != UCL_OBJECT) { @@ -1646,13 +1737,13 @@ ucl_inherit_handler (const unsigned char *data, size_t len, target = parser->stack->obj; - if (args && (cur = ucl_object_find_key (args, "replace")) != NULL) { + if (args && (cur = ucl_object_lookup (args, "replace")) != NULL) { replace = ucl_object_toboolean (cur); } - while ((cur = ucl_iterate_object (parent, &it, true))) { + while ((cur = ucl_object_iterate (parent, &it, true))) { /* We do not replace existing keys */ - if (!replace && ucl_object_find_keyl (target, cur->key, cur->keylen)) { + if (!replace && ucl_object_lookup_len (target, cur->key, cur->keylen)) { continue; } @@ -2097,7 +2188,7 @@ ucl_object_insert_key_common (ucl_object_t *top, ucl_object_t *elt, } else if (found->type == UCL_OBJECT && elt->type == UCL_OBJECT) { /* Mix two hashes */ - while ((cur = ucl_iterate_object (elt, &it, true)) != NULL) { + while ((cur = ucl_object_iterate (elt, &it, true)) != NULL) { tmp = ucl_object_ref (cur); ucl_object_insert_key_common (found, tmp, cur->key, cur->keylen, copy_key, false, false); @@ -2126,7 +2217,7 @@ ucl_object_delete_keyl (ucl_object_t *top, const char *key, size_t keylen) return false; } - found = __DECONST (ucl_object_t *, ucl_object_find_keyl (top, key, keylen)); + found = __DECONST (ucl_object_t *, ucl_object_lookup_len (top, key, keylen)); if (found == NULL) { return false; @@ -2153,7 +2244,7 @@ ucl_object_pop_keyl (ucl_object_t *top, const char *key, size_t keylen) if (top == NULL || key == NULL) { return false; } - found = ucl_object_find_keyl (top, key, keylen); + found = ucl_object_lookup_len (top, key, keylen); if (found == NULL) { return NULL; @@ -2226,7 +2317,7 @@ ucl_object_merge (ucl_object_t *top, ucl_object_t *elt, bool copy) } const ucl_object_t * -ucl_object_find_keyl (const ucl_object_t *obj, const char *key, size_t klen) +ucl_object_lookup_len (const ucl_object_t *obj, const char *key, size_t klen) { const ucl_object_t *ret; ucl_object_t srch; @@ -2243,17 +2334,17 @@ ucl_object_find_keyl (const ucl_object_t *obj, const char *key, size_t klen) } const ucl_object_t * -ucl_object_find_key (const ucl_object_t *obj, const char *key) +ucl_object_lookup (const ucl_object_t *obj, const char *key) { if (key == NULL) { return NULL; } - return ucl_object_find_keyl (obj, key, strlen (key)); + return ucl_object_lookup_len (obj, key, strlen (key)); } const ucl_object_t* -ucl_object_find_any_key (const ucl_object_t *obj, +ucl_object_lookup_any (const ucl_object_t *obj, const char *key, ...) { va_list ap; @@ -2264,7 +2355,7 @@ ucl_object_find_any_key (const ucl_object_t *obj, return NULL; } - ret = ucl_object_find_keyl (obj, key, strlen (key)); + ret = ucl_object_lookup_len (obj, key, strlen (key)); if (ret == NULL) { va_start (ap, key); @@ -2276,7 +2367,7 @@ ucl_object_find_any_key (const ucl_object_t *obj, break; } else { - ret = ucl_object_find_keyl (obj, nk, strlen (nk)); + ret = ucl_object_lookup_len (obj, nk, strlen (nk)); } } @@ -2287,7 +2378,7 @@ ucl_object_find_any_key (const ucl_object_t *obj, } const ucl_object_t* -ucl_iterate_object (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values) +ucl_object_iterate (const ucl_object_t *obj, ucl_object_iter_t *iter, bool expand_values) { const ucl_object_t *elt = NULL; @@ -2394,7 +2485,7 @@ ucl_object_iterate_safe (ucl_object_iter_t it, bool expand_values) } if (rit->impl_it->type == UCL_OBJECT || rit->impl_it->type == UCL_ARRAY) { - ret = ucl_iterate_object (rit->impl_it, &rit->expl_it, true); + ret = ucl_object_iterate (rit->impl_it, &rit->expl_it, true); if (ret == NULL) { /* Need to switch to another implicit object in chain */ @@ -2429,13 +2520,13 @@ ucl_object_iterate_free (ucl_object_iter_t it) } const ucl_object_t * -ucl_lookup_path (const ucl_object_t *top, const char *path_in) { - return ucl_lookup_path_char (top, path_in, '.'); +ucl_object_lookup_path (const ucl_object_t *top, const char *path_in) { + return ucl_object_lookup_path_char (top, path_in, '.'); } const ucl_object_t * -ucl_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) { +ucl_object_lookup_path_char (const ucl_object_t *top, const char *path_in, const char sep) { const ucl_object_t *o = NULL, *found; const char *p, *c; char *err_str; @@ -2468,7 +2559,7 @@ ucl_lookup_path_char (const ucl_object_t *top, const char *path_in, const char s o = ucl_array_find_index (top, index); break; default: - o = ucl_object_find_keyl (top, c, p - c); + o = ucl_object_lookup_len (top, c, p - c); break; } if (o == NULL) { @@ -2527,7 +2618,7 @@ ucl_object_new_full (ucl_type_t type, unsigned priority) } } else { - new = ucl_object_new_userdata (NULL, NULL); + new = ucl_object_new_userdata (NULL, NULL, NULL); ucl_object_set_priority (new, priority); } @@ -2535,7 +2626,9 @@ ucl_object_new_full (ucl_type_t type, unsigned priority) } ucl_object_t* -ucl_object_new_userdata (ucl_userdata_dtor dtor, ucl_userdata_emitter emitter) +ucl_object_new_userdata (ucl_userdata_dtor dtor, + ucl_userdata_emitter emitter, + void *ptr) { struct ucl_object_userdata *new; size_t nsize = sizeof (*new); @@ -2549,6 +2642,7 @@ ucl_object_new_userdata (ucl_userdata_dtor dtor, ucl_userdata_emitter emitter) new->obj.prev = (ucl_object_t *)new; new->dtor = dtor; new->emitter = emitter; + new->obj.value.ud = ptr; } return (ucl_object_t *)new; @@ -2691,14 +2785,16 @@ ucl_array_merge (ucl_object_t *top, ucl_object_t *elt, bool copy) UCL_ARRAY_GET (v1, top); UCL_ARRAY_GET (v2, cp); - kv_concat (ucl_object_t *, *v1, *v2); + if (v1 && v2) { + kv_concat (ucl_object_t *, *v1, *v2); - for (i = v2->n; i < v1->n; i ++) { - obj = &kv_A (*v1, i); - if (*obj == NULL) { - continue; + for (i = v2->n; i < v1->n; i ++) { + obj = &kv_A (*v1, i); + if (*obj == NULL) { + continue; + } + top->len ++; } - top->len ++; } return true; @@ -3087,7 +3183,7 @@ ucl_object_copy_internal (const ucl_object_t *other, bool allow_array) /* reset old value */ memset (&new->value, 0, sizeof (new->value)); - while ((cur = ucl_iterate_object (other, &it, true)) != NULL) { + while ((cur = ucl_object_iterate (other, &it, true)) != NULL) { if (other->type == UCL_ARRAY) { ucl_array_append (new, ucl_object_copy_internal (cur, false)); } @@ -3193,8 +3289,8 @@ ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2) break; case UCL_OBJECT: if (o1->len == o2->len && o1->len > 0) { - while ((it1 = ucl_iterate_object (o1, &iter, true)) != NULL) { - it2 = ucl_object_find_key (o2, ucl_object_key (it1)); + while ((it1 = ucl_object_iterate (o1, &iter, true)) != NULL) { + it2 = ucl_object_lookup (o2, ucl_object_key (it1)); if (it2 == NULL) { ret = 1; break; @@ -3217,6 +3313,13 @@ ucl_object_compare (const ucl_object_t *o1, const ucl_object_t *o2) return ret; } +int +ucl_object_compare_qsort (const ucl_object_t **o1, + const ucl_object_t **o2) +{ + return ucl_object_compare (*o1, *o2); +} + void ucl_object_array_sort (ucl_object_t *ar, int (*cmp)(const ucl_object_t **o1, const ucl_object_t **o2)) @@ -3255,3 +3358,131 @@ ucl_object_set_priority (ucl_object_t *obj, obj->flags = priority; } } + +bool +ucl_object_string_to_type (const char *input, ucl_type_t *res) +{ + if (strcasecmp (input, "object") == 0) { + *res = UCL_OBJECT; + } + else if (strcasecmp (input, "array") == 0) { + *res = UCL_ARRAY; + } + else if (strcasecmp (input, "integer") == 0) { + *res = UCL_INT; + } + else if (strcasecmp (input, "number") == 0) { + *res = UCL_FLOAT; + } + else if (strcasecmp (input, "string") == 0) { + *res = UCL_STRING; + } + else if (strcasecmp (input, "boolean") == 0) { + *res = UCL_BOOLEAN; + } + else if (strcasecmp (input, "null") == 0) { + *res = UCL_NULL; + } + else if (strcasecmp (input, "userdata") == 0) { + *res = UCL_USERDATA; + } + else { + return false; + } + + return true; +} + +const char * +ucl_object_type_to_string (ucl_type_t type) +{ + const char *res = "unknown"; + + switch (type) { + case UCL_OBJECT: + res = "object"; + break; + case UCL_ARRAY: + res = "array"; + break; + case UCL_INT: + res = "integer"; + break; + case UCL_FLOAT: + case UCL_TIME: + res = "number"; + break; + case UCL_STRING: + res = "string"; + break; + case UCL_BOOLEAN: + res = "boolean"; + break; + case UCL_USERDATA: + res = "userdata"; + break; + case UCL_NULL: + res = "null"; + break; + } + + return res; +} + +const ucl_object_t * +ucl_parser_get_comments (struct ucl_parser *parser) +{ + if (parser && parser->comments) { + return parser->comments; + } + + return NULL; +} + +const ucl_object_t * +ucl_comments_find (const ucl_object_t *comments, + const ucl_object_t *srch) +{ + if (comments && srch) { + return ucl_object_lookup_len (comments, (const char *)&srch, + sizeof (void *)); + } + + return NULL; +} + +bool +ucl_comments_move (ucl_object_t *comments, + const ucl_object_t *from, const ucl_object_t *to) +{ + const ucl_object_t *found; + ucl_object_t *obj; + + if (comments && from && to) { + found = ucl_object_lookup_len (comments, + (const char *)&from, sizeof (void *)); + + if (found) { + /* Replace key */ + obj = ucl_object_ref (found); + ucl_object_delete_keyl (comments, (const char *)&from, + sizeof (void *)); + ucl_object_insert_key (comments, obj, (const char *)&to, + sizeof (void *), true); + + return true; + } + } + + return false; +} + +void +ucl_comments_add (ucl_object_t *comments, const ucl_object_t *obj, + const char *comment) +{ + if (comments && obj && comment) { + ucl_object_insert_key (comments, ucl_object_fromstring (comment), + (const char *)&obj, sizeof (void *), true); + } +} diff --git a/contrib/libucl/tests/basic.test b/contrib/libucl/tests/basic.test index a4a07eb13ad9..174cc8c3f0f1 100755 --- a/contrib/libucl/tests/basic.test +++ b/contrib/libucl/tests/basic.test @@ -21,6 +21,18 @@ for _tin in ${TEST_DIR}/basic/*.in ; do fi fi rm $_out + # Use FD interface + $PROG -f $_t.in > /dev/null + # JSON output + $PROG -j $_t.in > /dev/null + $PROG -c -j $_t.in > /dev/null + # YAML output + $PROG -y $_t.in > /dev/null + # Save comments mode + $PROG -C $_t.in > /dev/null + # Save macro mode + $PROG -M $_t.in > /dev/null + $PROG -M -C $_t.in > /dev/null done diff --git a/contrib/libucl/tests/basic/18.in b/contrib/libucl/tests/basic/18.in index bd12c019859a..9b5655afc25a 100644 --- a/contrib/libucl/tests/basic/18.in +++ b/contrib/libucl/tests/basic/18.in @@ -7,4 +7,25 @@ defaults { mything { .inherit "defaults" key = "newval" + key = "newval1" +} +mything { + .inherit "mything" + key = "newval" +} +.priority 3 + +defaults { + key = "val1" + foo = "bar1" + many = "values here" +} +mything1 { + key2 = "wtf??" + .priority 1 + .inherit "defaults" + .inherit "mything" + .inherit "mything1" + key1 = "newval" + key2 = "OMG" # low priority } diff --git a/contrib/libucl/tests/basic/18.res b/contrib/libucl/tests/basic/18.res index a6272d508ac5..92bf0f8e8840 100644 --- a/contrib/libucl/tests/basic/18.res +++ b/contrib/libucl/tests/basic/18.res @@ -1,5 +1,11 @@ defaults { - key = "val"; + key = "val1"; + foo = "bar1"; + many = "values here"; +} +mything { + key = "newval"; + key = "newval1"; foo = "bar"; many = "values here"; } @@ -8,4 +14,11 @@ mything { foo = "bar"; many = "values here"; } +mything1 { + key2 = "wtf??"; + key = "val1"; + foo = "bar1"; + many = "values here"; + key1 = "newval"; +} diff --git a/contrib/libucl/tests/basic/2.res b/contrib/libucl/tests/basic/2.res index 72e95705ec46..cac7a243ccc6 100644 --- a/contrib/libucl/tests/basic/2.res +++ b/contrib/libucl/tests/basic/2.res @@ -29,7 +29,7 @@ key4 = 5000000; key4 = "s1"; key5 = 0.010000; key5 = "\n\r123"; -key6 = 2207520000.000000; +key6 = 315360000.0; keyvar = "unknowntest"; keyvar = "unknownunknownunknown${unknown}"; keyvar = "${some}$no${}$$test$$$$$$$"; diff --git a/contrib/libucl/tests/basic/9.in b/contrib/libucl/tests/basic/9.in index 4c06e39dcaeb..2341445edbf0 100644 --- a/contrib/libucl/tests/basic/9.in +++ b/contrib/libucl/tests/basic/9.in @@ -6,7 +6,18 @@ key = value; .include "$CURDIR/9.inc" -.try_include "/non/existent" #.try_include "$CURDIR/9.incorrect.inc" # 9.incorrect.inc contains '{}}' #key = value; +prefix1 = { + key = value +} +array = [10] +array1 = [10] +.include(prefix=true; key="prefix") "$CURDIR/9.inc" +.include(prefix=true; key="prefix2"; target="array"; glob=true) "$CURDIR/9.inc" +.include(prefix=true; key="prefix1"; target="array"; glob=true) "$CURDIR/9.inc" +.include(prefix=true; key="array"; target="array"; glob=true) "$CURDIR/9.inc" +.include(prefix=true; key="array1"; glob=true) "$CURDIR/9.inc" +.include(prefix=true; key="prefix"; glob=true) "$CURDIR/9.inc" +.try_include "/non/existent" diff --git a/contrib/libucl/tests/basic/9.res b/contrib/libucl/tests/basic/9.res index ec3f014e60d4..0ed36e84422a 100644 --- a/contrib/libucl/tests/basic/9.res +++ b/contrib/libucl/tests/basic/9.res @@ -2,4 +2,33 @@ key1 = "value"; key1 = "value"; key1 = "value"; key = "value"; +prefix1 [ + { + key = "value"; + } + { + key1 = "value"; + } +] +array [ + 10, + { + key1 = "value"; + } +] +array1 [ + 10, + { + key1 = "value"; + } +] +prefix { + key1 = "value"; + key1 = "value"; +} +prefix2 [ + { + key1 = "value"; + } +] diff --git a/contrib/libucl/tests/basic/escapes.in b/contrib/libucl/tests/basic/escapes.in new file mode 100644 index 000000000000..d9da67a3fd53 --- /dev/null +++ b/contrib/libucl/tests/basic/escapes.in @@ -0,0 +1,2 @@ +# Checks for escapes in strings +str = "\r\n\b\t\f\\\"\u03B4\u0B90\u1F640\uFFFFsome text" diff --git a/contrib/libucl/tests/basic/escapes.res b/contrib/libucl/tests/basic/escapes.res new file mode 100644 index 000000000000..e914e778ff7d --- /dev/null +++ b/contrib/libucl/tests/basic/escapes.res @@ -0,0 +1,2 @@ +str = "\r\n\b\t\f\\\"δஐὤ0￿some text"; + diff --git a/contrib/libucl/tests/basic/load.in b/contrib/libucl/tests/basic/load.in new file mode 100644 index 000000000000..3ed4cee9a0d9 --- /dev/null +++ b/contrib/libucl/tests/basic/load.in @@ -0,0 +1,14 @@ +# Load macro tests +section { + +.load(try=false, multiline=false, trim=false, escape=false, key="key1", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=true, trim=false, escape=false, key="key2", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=true, trim=true, escape=false, key="key3", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=true, trim=true, escape=true, key="key4", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=false, trim=true, escape=false, key="key5", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=false, trim=false, escape=true, key="key6", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=false, trim=true, escape=true, key="key7", target="string", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=false, trim=false, escape=false, key="key8", target="int", priority=1) "${CURDIR}/load.inc" +.load(try=false, multiline=false, trim=false, escape=false, key="key9", target="int", priority=4) "${CURDIR}/load.inc" +.load(try=true, multiline=false, trim=false, escape=false, key="key10", target="string", priority=1) "load_bad.inc" +}; diff --git a/contrib/libucl/tests/basic/load.inc b/contrib/libucl/tests/basic/load.inc new file mode 100644 index 000000000000..cbd9a13b82a2 --- /dev/null +++ b/contrib/libucl/tests/basic/load.inc @@ -0,0 +1,2 @@ + 123 + 321\n diff --git a/contrib/libucl/tests/basic/load.res b/contrib/libucl/tests/basic/load.res new file mode 100644 index 000000000000..9ab1eac6b178 --- /dev/null +++ b/contrib/libucl/tests/basic/load.res @@ -0,0 +1,19 @@ +section { + key1 = " 123\n 321\\n \n"; + key2 = <> /tmp/_ucl_test_schema.out < $i && ( echo "OK" ) || ( echo "Fail" ; exit 1 ) + $PROG >> /tmp/_ucl_test_schema.out < $i + if [ $? -eq 0 ] ; then + echo "OK" + _succeed=$(($_succeed + 1)) + else + echo "Fail" + fi + _tests=$(($_tests + 1)) done + +if [ $_tests -ne $_succeed ] ; then + exit 1 +fi diff --git a/contrib/libucl/tests/schema/definitions.json b/contrib/libucl/tests/schema/definitions.json index cf935a321532..1ab9b2163c44 100644 --- a/contrib/libucl/tests/schema/definitions.json +++ b/contrib/libucl/tests/schema/definitions.json @@ -1,7 +1,7 @@ [ { "description": "valid definition", - "schema": {"$ref": "http://json-schema.org/draft-04/schema#"}, + "schema": {"$ref": "http://highsecure.ru/ucl-schema/schema#"}, "tests": [ { "description": "valid definition schema", @@ -16,7 +16,7 @@ }, { "description": "invalid definition", - "schema": {"$ref": "http://json-schema.org/draft-04/schema#"}, + "schema": {"$ref": "http://highsecure.ru/ucl-schema/schema#"}, "tests": [ { "description": "invalid definition schema", diff --git a/contrib/libucl/tests/schema/ref.json b/contrib/libucl/tests/schema/ref.json index 1b88b27d6faf..1767769cd845 100644 --- a/contrib/libucl/tests/schema/ref.json +++ b/contrib/libucl/tests/schema/ref.json @@ -125,10 +125,9 @@ } ] }, -/* { "description": "remote ref, containing refs itself", - "schema": {"$ref": "http://json-schema.org/draft-04/schema#"}, + "schema": {"$ref": "http://highsecure.ru/ucl-schema/schema#"}, "tests": [ { "description": "remote ref valid", @@ -142,5 +141,4 @@ } ] } -*/ ] diff --git a/contrib/libucl/tests/schema/refRemote.json b/contrib/libucl/tests/schema/refRemote.json index 4ca804732c97..0ba9adb94ca1 100644 --- a/contrib/libucl/tests/schema/refRemote.json +++ b/contrib/libucl/tests/schema/refRemote.json @@ -1,7 +1,7 @@ [ { "description": "remote ref", - "schema": {"$ref": "http://localhost:1234/integer.json"}, + "schema": {"$ref": "http://highsecure.ru/ucl-schema/remotes/integer.json"}, "tests": [ { "description": "remote ref valid", @@ -17,7 +17,7 @@ }, { "description": "fragment within remote ref", - "schema": {"$ref": "http://localhost:1234/subSchemas.json#/integer"}, + "schema": {"$ref": "http://highsecure.ru/ucl-schema/remotes/subSchemas.json#/integer"}, "tests": [ { "description": "remote fragment valid", @@ -34,7 +34,7 @@ { "description": "ref within remote ref", "schema": { - "$ref": "http://localhost:1234/subSchemas.json#/refToInteger" + "$ref": "http://highsecure.ru/ucl-schema/remotes/subSchemas.json#/refToInteger" }, "tests": [ { @@ -49,10 +49,11 @@ } ] }, +/* { "description": "change resolution scope", "schema": { - "id": "http://localhost:1234/", + "id": "http://highsecure.ru/ucl-schema/remotes/", "items": { "id": "folder/", "items": {"$ref": "folderInteger.json"} @@ -71,4 +72,5 @@ } ] } +*/ ] diff --git a/contrib/libucl/tests/test_basic.c b/contrib/libucl/tests/test_basic.c index 45a9c8b2bc6a..b7acc73b4a69 100644 --- a/contrib/libucl/tests/test_basic.c +++ b/contrib/libucl/tests/test_basic.c @@ -23,20 +23,27 @@ #include "ucl.h" #include "ucl_internal.h" +#include +#include +#include + int main (int argc, char **argv) { - char *inbuf; + char *inbuf = NULL; struct ucl_parser *parser = NULL, *parser2 = NULL; - ucl_object_t *obj; + ucl_object_t *obj, *comments = NULL; ssize_t bufsize, r; FILE *in, *out; unsigned char *emitted = NULL; const char *fname_in = NULL, *fname_out = NULL; - int ret = 0, opt, json = 0, compact = 0, yaml = 0; + int ret = 0, opt, json = 0, compact = 0, yaml = 0, + save_comments = 0, skip_macro = 0, + flags, fd_out, fd_in, use_fd = 0; + struct ucl_emitter_functions *func; - while ((opt = getopt(argc, argv, "jcy")) != -1) { + while ((opt = getopt(argc, argv, "fjcyCM")) != -1) { switch (opt) { case 'j': json = 1; @@ -44,11 +51,20 @@ main (int argc, char **argv) case 'c': compact = 1; break; + case 'C': + save_comments = 1; + break; case 'y': yaml = 1; break; + case 'M': + skip_macro = true; + break; + case 'f': + use_fd = true; + break; default: /* '?' */ - fprintf (stderr, "Usage: %s [-jcy] [in] [out]\n", + fprintf (stderr, "Usage: %s [-jcy] [-CM] [-f] [in] [out]\n", argv[0]); exit (EXIT_FAILURE); } @@ -67,64 +83,113 @@ main (int argc, char **argv) break; } - if (fname_in != NULL) { - in = fopen (fname_in, "r"); - if (in == NULL) { - exit (-errno); + if (!use_fd) { + if (fname_in != NULL) { + in = fopen (fname_in, "r"); + if (in == NULL) { + exit (-errno); + } + } + else { + in = stdin; } } else { - in = stdin; + if (fname_in != NULL) { + fd_in = open (fname_in, O_RDONLY); + if (fd_in == -1) { + exit (-errno); + } + } + else { + fd_in = STDIN_FILENO; + } } - parser = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE); + + flags = UCL_PARSER_KEY_LOWERCASE; + + if (save_comments) { + flags |= UCL_PARSER_SAVE_COMMENTS; + } + + if (skip_macro) { + flags |= UCL_PARSER_DISABLE_MACRO; + } + + parser = ucl_parser_new (flags); ucl_parser_register_variable (parser, "ABI", "unknown"); if (fname_in != NULL) { ucl_parser_set_filevars (parser, fname_in, true); } - inbuf = malloc (BUFSIZ); - bufsize = BUFSIZ; - r = 0; + if (!use_fd) { + inbuf = malloc (BUFSIZ); + bufsize = BUFSIZ; + r = 0; - while (!feof (in) && !ferror (in)) { - if (r == bufsize) { - inbuf = realloc (inbuf, bufsize * 2); - bufsize *= 2; - if (inbuf == NULL) { - perror ("realloc"); - exit (EXIT_FAILURE); + while (!feof (in) && !ferror (in)) { + if (r == bufsize) { + inbuf = realloc (inbuf, bufsize * 2); + bufsize *= 2; + if (inbuf == NULL) { + perror ("realloc"); + exit (EXIT_FAILURE); + } + } + r += fread (inbuf + r, 1, bufsize - r, in); + } + + if (ferror (in)) { + fprintf (stderr, "Failed to read the input file.\n"); + exit (EXIT_FAILURE); + } + + ucl_parser_add_chunk (parser, (const unsigned char *)inbuf, r); + fclose (in); + } + else { + ucl_parser_add_fd (parser, fd_in); + close (fd_in); + } + + if (!use_fd) { + if (fname_out != NULL) { + out = fopen (fname_out, "w"); + if (out == NULL) { + exit (-errno); } } - r += fread (inbuf + r, 1, bufsize - r, in); - } - - if (ferror (in)) { - fprintf (stderr, "Failed to read the input file.\n"); - exit (EXIT_FAILURE); - } - - ucl_parser_add_chunk (parser, (const unsigned char *)inbuf, r); - fclose (in); - - if (fname_out != NULL) { - out = fopen (fname_out, "w"); - if (out == NULL) { - exit (-errno); + else { + out = stdout; } } else { - out = stdout; + if (fname_out != NULL) { + fd_out = open (fname_out, O_WRONLY | O_CREAT, 00644); + if (fd_out == -1) { + exit (-errno); + } + } + else { + fd_out = STDOUT_FILENO; + } } + if (ucl_parser_get_error (parser) != NULL) { - fprintf (out, "Error occurred: %s\n", ucl_parser_get_error(parser)); + fprintf (out, "Error occurred (phase 1): %s\n", + ucl_parser_get_error(parser)); ret = 1; goto end; } obj = ucl_parser_get_object (parser); + if (save_comments) { + comments = ucl_object_ref (ucl_parser_get_comments (parser)); + } + if (json) { if (compact) { emitted = ucl_object_emit (obj, UCL_EMIT_JSON_COMPACT); @@ -137,16 +202,27 @@ main (int argc, char **argv) emitted = ucl_object_emit (obj, UCL_EMIT_YAML); } else { - emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG); + emitted = NULL; + func = ucl_object_emit_memory_funcs ((void **)&emitted); + + if (func != NULL) { + ucl_object_emit_full (obj, UCL_EMIT_CONFIG, func, comments); + ucl_object_emit_funcs_free (func); + } } +#if 0 + fprintf (out, "%s\n****\n", emitted); +#endif + ucl_parser_free (parser); ucl_object_unref (obj); - parser2 = ucl_parser_new (UCL_PARSER_KEY_LOWERCASE); + parser2 = ucl_parser_new (flags); ucl_parser_add_string (parser2, (const char *)emitted, 0); if (ucl_parser_get_error(parser2) != NULL) { - fprintf (out, "Error occurred: %s\n", ucl_parser_get_error(parser2)); + fprintf (out, "Error occurred (phase 2): %s\n", + ucl_parser_get_error(parser2)); fprintf (out, "%s\n", emitted); ret = 1; goto end; @@ -155,38 +231,68 @@ main (int argc, char **argv) if (emitted != NULL) { free (emitted); } + if (comments) { + ucl_object_unref (comments); + comments = NULL; + } + + if (save_comments) { + comments = ucl_object_ref (ucl_parser_get_comments (parser2)); + } obj = ucl_parser_get_object (parser2); - if (json) { - if (compact) { - emitted = ucl_object_emit (obj, UCL_EMIT_JSON_COMPACT); - } - else { - emitted = ucl_object_emit (obj, UCL_EMIT_JSON); - } - } - else if (yaml) { - emitted = ucl_object_emit (obj, UCL_EMIT_YAML); + + if (!use_fd) { + func = ucl_object_emit_file_funcs (out); } else { - emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG); + func = ucl_object_emit_fd_funcs (fd_out); + } + + if (func != NULL) { + if (json) { + if (compact) { + ucl_object_emit_full (obj, UCL_EMIT_JSON_COMPACT, + func, comments); + } + else { + ucl_object_emit_full (obj, UCL_EMIT_JSON, + func, comments); + } + } + else if (yaml) { + ucl_object_emit_full (obj, UCL_EMIT_YAML, + func, comments); + } + else { + ucl_object_emit_full (obj, UCL_EMIT_CONFIG, + func, comments); + } + + ucl_object_emit_funcs_free (func); + } + + if (!use_fd) { + fprintf (out, "\n"); + fclose (out); + } + else { + write (fd_out, "\n", 1); + close (fd_out); } - fprintf (out, "%s\n", emitted); ucl_object_unref (obj); end: - if (emitted != NULL) { - free (emitted); - } if (parser2 != NULL) { ucl_parser_free (parser2); } + if (comments) { + ucl_object_unref (comments); + } if (inbuf != NULL) { free (inbuf); } - fclose (out); - return ret; } diff --git a/contrib/libucl/tests/test_generate.c b/contrib/libucl/tests/test_generate.c index f09eabb839fe..a8dc2fafc2bb 100644 --- a/contrib/libucl/tests/test_generate.c +++ b/contrib/libucl/tests/test_generate.c @@ -26,15 +26,29 @@ #include #include "ucl.h" +static void +ud_dtor (void *ptr) +{ + assert (ptr == NULL); +} + +static const char * +ud_emit (void *ptr) +{ + return "test userdata emit"; +} + int main (int argc, char **argv) { - ucl_object_t *obj, *cur, *ar, *ref; + ucl_object_t *obj, *cur, *ar, *ar1, *ref, *test_obj, *comments; ucl_object_iter_t it; - const ucl_object_t *found, *it_obj; + const ucl_object_t *found, *it_obj, *test; + struct ucl_emitter_functions *fn; FILE *out; unsigned char *emitted; const char *fname_out = NULL; + struct ucl_parser *parser; int ret = 0; switch (argc) { @@ -65,7 +79,8 @@ main (int argc, char **argv) /* Create some strings */ cur = ucl_object_fromstring_common (" test string ", 0, UCL_STRING_TRIM); ucl_object_insert_key (obj, cur, "key1", 0, false); - cur = ucl_object_fromstring_common (" test \nstring\n ", 0, UCL_STRING_TRIM | UCL_STRING_ESCAPE); + cur = ucl_object_fromstring_common (" test \nstring\n\r\n\b\t\f\\\" ", 0, + UCL_STRING_TRIM | UCL_STRING_ESCAPE); ucl_object_insert_key (obj, cur, "key2", 0, false); cur = ucl_object_fromstring_common (" test string \n", 0, 0); ucl_object_insert_key (obj, cur, "key3", 0, false); @@ -73,10 +88,34 @@ main (int argc, char **argv) ar = ucl_object_typed_new (UCL_ARRAY); cur = ucl_object_fromint (10); ucl_array_append (ar, cur); + assert (ucl_array_index_of (ar, cur) == 0); cur = ucl_object_fromdouble (10.1); ucl_array_append (ar, cur); + assert (ucl_array_index_of (ar, cur) == 1); cur = ucl_object_fromdouble (9.999); ucl_array_prepend (ar, cur); + assert (ucl_array_index_of (ar, cur) == 0); + + ar1 = ucl_object_copy (ar); + cur = ucl_object_fromstring ("abc"); + ucl_array_prepend (ar1, cur); + cur = ucl_object_fromstring ("cde"); + ucl_array_prepend (ar1, cur); + cur = ucl_object_fromstring ("абв"); /* UTF8 */ + ucl_array_prepend (ar1, cur); + cur = ucl_object_fromstring ("Ебв"); /* UTF8 */ + ucl_array_prepend (ar1, cur); +/* + * This is ususally broken or fragile as utf collate is far from perfect + cur = ucl_object_fromstring ("ёбв"); + ucl_array_prepend (ar1, cur); + cur = ucl_object_fromstring ("Ёбв"); // hello to @bapt + ucl_array_prepend (ar1, cur); +*/ + cur = ucl_object_fromstring ("😎"); /* everybody likes emoji in the code */ + ucl_array_prepend (ar1, cur); + + ucl_object_array_sort (ar1, ucl_object_compare_qsort); /* Removing from an array */ cur = ucl_object_fromdouble (1.0); @@ -122,22 +161,80 @@ main (int argc, char **argv) ucl_object_insert_key (obj, cur, "key13", 0, false); cur = ucl_object_frombool (true); ucl_object_insert_key (obj, cur, "k=3", 0, false); + ucl_object_insert_key (obj, ar1, "key14", 0, false); + cur = ucl_object_new_userdata (ud_dtor, ud_emit, NULL); + ucl_object_insert_key (obj, cur, "key15", 0, false); + + /* More tests for keys */ + cur = ucl_object_fromlstring ("test", 3); + ucl_object_insert_key (obj, cur, "key16", 0, false); + test = ucl_object_lookup_any (obj, "key100", "key200", "key300", "key16", NULL); + assert (test == cur); + test = ucl_object_lookup_len (obj, "key160", 5); + assert (test == cur); + cur = ucl_object_pop_key (obj, "key16"); + assert (test == cur); + test = ucl_object_pop_key (obj, "key16"); + assert (test == NULL); + test = ucl_object_lookup_len (obj, "key160", 5); + assert (test == NULL); + /* Objects merging tests */ + test_obj = ucl_object_new_full (UCL_OBJECT, 2); + ucl_object_insert_key (test_obj, cur, "key16", 0, true); + ucl_object_merge (obj, test_obj, true); + ucl_object_unref (test_obj); + /* Array merging test */ + test_obj = ucl_object_new_full (UCL_ARRAY, 3); + ucl_array_append (test_obj, ucl_object_fromstring ("test")); + ucl_array_merge (test_obj, ar1, false); + ucl_object_insert_key (obj, test_obj, "key17", 0, true); + /* Object deletion */ + cur = ucl_object_fromstring ("test"); + ucl_object_insert_key (obj, cur, "key18", 0, true); + assert (ucl_object_delete_key (obj, "key18")); + assert (!ucl_object_delete_key (obj, "key18")); + cur = ucl_object_fromlstring ("test", 4); + ucl_object_insert_key (obj, cur, "key18\0\0", 7, true); + assert (ucl_object_lookup_len (obj, "key18\0\0", 7) == cur); + assert (ucl_object_lookup (obj, "key18") == NULL); + assert (ucl_object_lookup_len (obj, "key18\0\1", 7) == NULL); + assert (ucl_object_delete_keyl (obj, "key18\0\0", 7)); + + /* Comments */ + + comments = ucl_object_typed_new (UCL_OBJECT); + found = ucl_object_lookup (obj, "key17"); + test = ucl_object_lookup (obj, "key16"); + ucl_comments_add (comments, found, "# test comment"); + assert (ucl_comments_find (comments, found) != NULL); + assert (ucl_comments_find (comments, test) == NULL); + ucl_comments_move (comments, found, test); + assert (ucl_comments_find (comments, found) == NULL); + assert (ucl_comments_find (comments, test) != NULL); + + /* Array replace */ + ar1 = ucl_object_typed_new (UCL_ARRAY); + cur = ucl_object_fromstring ("test"); + cur = ucl_elt_append (cur, ucl_object_fromstring ("test1")); + ucl_array_append (ar1, cur); + test = ucl_array_replace_index (ar1, ucl_object_fromstring ("test2"), 0); + assert (test == cur); /* Try to find using path */ /* Should exist */ - found = ucl_lookup_path (obj, "key4.1"); + found = ucl_object_lookup_path (obj, "key4.1"); assert (found != NULL && ucl_object_toint (found) == 10); /* . should be ignored */ - found = ucl_lookup_path (obj, ".key4.1"); + found = ucl_object_lookup_path (obj, ".key4.1"); assert (found != NULL && ucl_object_toint (found) == 10); /* moar dots... */ - found = ucl_lookup_path (obj, ".key4........1..."); + found = ucl_object_lookup_path (obj, ".key4........1..."); assert (found != NULL && ucl_object_toint (found) == 10); /* No such index */ - found = ucl_lookup_path (obj, ".key4.3"); + found = ucl_object_lookup_path (obj, ".key4.3"); assert (found == NULL); /* No such key */ - found = ucl_lookup_path (obj, "key9..key1"); + found = ucl_object_lookup_path (obj, "key9..key1"); assert (found == NULL); /* Test iteration */ @@ -167,10 +264,36 @@ main (int argc, char **argv) assert (ucl_object_type (it_obj) == UCL_BOOLEAN); ucl_object_iterate_free (it); - emitted = ucl_object_emit (obj, UCL_EMIT_CONFIG); - + fn = ucl_object_emit_memory_funcs (&emitted); + assert (ucl_object_emit_full (obj, UCL_EMIT_CONFIG, fn, comments)); fprintf (out, "%s\n", emitted); + ucl_object_emit_funcs_free (fn); ucl_object_unref (obj); + ucl_object_unref (comments); + + parser = ucl_parser_new (UCL_PARSER_NO_IMPLICIT_ARRAYS); + + if (ucl_parser_add_chunk_full (parser, emitted, strlen (emitted), + 3, UCL_DUPLICATE_ERROR, UCL_PARSE_UCL)) { + /* Should fail due to duplicate */ + assert (0); + } + else { + assert (ucl_parser_get_error (parser) != NULL); + ucl_parser_clear_error (parser); + ucl_parser_free (parser); + parser = ucl_parser_new (0); + ucl_parser_add_chunk_full (parser, emitted, strlen (emitted), + 3, UCL_DUPLICATE_MERGE, UCL_PARSE_UCL); + } + + assert (ucl_parser_get_column (parser) == 0); + assert (ucl_parser_get_linenum (parser) != 0); + ucl_parser_clear_error (parser); + assert (ucl_parser_get_error_code (parser) == 0); + obj = ucl_parser_get_object (parser); + ucl_parser_free (parser); + ucl_object_free (obj); if (emitted != NULL) { free (emitted); diff --git a/contrib/libucl/tests/test_msgpack.c b/contrib/libucl/tests/test_msgpack.c index dd5860e9828b..00f804a0a07e 100644 --- a/contrib/libucl/tests/test_msgpack.c +++ b/contrib/libucl/tests/test_msgpack.c @@ -26,9 +26,9 @@ #include "ucl_internal.h" #include -static const int niter = 1000; -static const int ntests = 100; -static const int nelt = 10; +static const int niter = 20; +static const int ntests = 10; +static const int nelt = 20; static int recursion = 0; @@ -39,6 +39,10 @@ static ucl_object_t* ucl_test_string (void); static ucl_object_t* ucl_test_boolean (void); static ucl_object_t* ucl_test_map (void); static ucl_object_t* ucl_test_array (void); +static ucl_object_t* ucl_test_large_map (void); +static ucl_object_t* ucl_test_large_array (void); +static ucl_object_t* ucl_test_large_string (void); +static ucl_object_t* ucl_test_null (void); ucl_msgpack_test tests[] = { ucl_test_integer, @@ -46,6 +50,7 @@ ucl_msgpack_test tests[] = { ucl_test_boolean, ucl_test_map, ucl_test_array, + ucl_test_null }; #define NTESTS (sizeof(tests) / sizeof(tests[0])) @@ -144,6 +149,18 @@ main (int argc, char **argv) ucl_object_insert_key (obj, elt, key, klen, true); } + key = random_key (&klen); + elt = ucl_test_large_array (); + ucl_object_insert_key (obj, elt, key, klen, true); + + key = random_key (&klen); + elt = ucl_test_large_map (); + ucl_object_insert_key (obj, elt, key, klen, true); + + key = random_key (&klen); + elt = ucl_test_large_string (); + ucl_object_insert_key (obj, elt, key, klen, true); + emitted = ucl_object_emit_len (obj, UCL_EMIT_MSGPACK, &elen); assert (emitted != NULL); @@ -189,6 +206,7 @@ ucl_test_integer (void) ucl_object_t *res; int count, i; uint64_t cur; + double curf; res = ucl_object_typed_new (UCL_ARRAY); count = pcg32_random () % nelt; @@ -196,13 +214,32 @@ ucl_test_integer (void) for (i = 0; i < count; i ++) { cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); ucl_array_append (res, ucl_object_fromint (cur % 128)); + ucl_array_append (res, ucl_object_fromint (-(cur % 128))); cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); - ucl_array_append (res, ucl_object_fromint (-cur % 128)); + ucl_array_append (res, ucl_object_fromint (cur % UINT16_MAX)); + ucl_array_append (res, ucl_object_fromint (-(cur % INT16_MAX))); cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); - ucl_array_append (res, ucl_object_fromint (cur % 65536)); + ucl_array_append (res, ucl_object_fromint (cur % UINT32_MAX)); + ucl_array_append (res, ucl_object_fromint (-(cur % INT32_MAX))); cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); - ucl_array_append (res, ucl_object_fromint (cur % INT32_MAX)); + ucl_array_append (res, ucl_object_fromint (cur)); + ucl_array_append (res, ucl_object_fromint (-cur)); + /* Double version */ cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); + curf = (cur % 128) / 19 * 16; + ucl_array_append (res, ucl_object_fromdouble (curf)); + cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); + curf = -(cur % 128) / 19 * 16; + ucl_array_append (res, ucl_object_fromdouble (curf)); + cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); + curf = (cur % 65536) / 19 * 16; + ucl_array_append (res, ucl_object_fromdouble (curf)); + ucl_array_append (res, ucl_object_fromdouble (-curf)); + cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); + curf = (cur % INT32_MAX) / 19 * 16; + ucl_array_append (res, ucl_object_fromdouble (curf)); + cur = ((uint64_t)pcg32_random ()) << 32 | pcg32_random (); + memcpy (&curf, &cur, sizeof (curf)); ucl_array_append (res, ucl_object_fromint (cur)); } @@ -251,6 +288,14 @@ ucl_test_string (void) free (str); } + /* One large string */ + str = malloc (65537); + elt = ucl_object_fromstring_common (str, 65537, + UCL_STRING_RAW); + elt->flags |= UCL_OBJECT_BINARY; + ucl_array_append (res, elt); + free (str); + return res; } @@ -287,7 +332,13 @@ ucl_test_map (void) for (i = 0; i < count; i ++) { if (recursion > 10) { - sel = pcg32_random () % (NTESTS - 2); + for (;;) { + sel = pcg32_random () % NTESTS; + if (tests[sel] != ucl_test_map && + tests[sel] != ucl_test_array) { + break; + } + } } else { sel = pcg32_random () % NTESTS; @@ -310,6 +361,32 @@ ucl_test_map (void) return res; } +static ucl_object_t* +ucl_test_large_map (void) +{ + ucl_object_t *res, *cur; + int count, i; + uint32_t cur_len; + size_t klen; + const char *key; + + res = ucl_object_typed_new (UCL_OBJECT); + count = 65537; + + recursion ++; + + for (i = 0; i < count; i ++) { + key = random_key (&klen); + cur = ucl_test_boolean (); + assert (cur != NULL); + assert (klen != 0); + + ucl_object_insert_key (res, cur, key, klen, true); + } + + return res; +} + static ucl_object_t* ucl_test_array (void) { @@ -324,7 +401,13 @@ ucl_test_array (void) for (i = 0; i < count; i ++) { if (recursion > 10) { - sel = pcg32_random () % (NTESTS - 2); + for (;;) { + sel = pcg32_random () % NTESTS; + if (tests[sel] != ucl_test_map && + tests[sel] != ucl_test_array) { + break; + } + } } else { sel = pcg32_random () % NTESTS; @@ -338,3 +421,47 @@ ucl_test_array (void) return res; } + +static ucl_object_t* +ucl_test_large_array (void) +{ + ucl_object_t *res, *cur; + int count, i; + uint32_t cur_len; + + res = ucl_object_typed_new (UCL_ARRAY); + count = 65537; + + recursion ++; + + for (i = 0; i < count; i ++) { + cur = ucl_test_boolean (); + assert (cur != NULL); + + ucl_array_append (res, cur); + } + + return res; +} + +static ucl_object_t* +ucl_test_large_string (void) +{ + ucl_object_t *res; + char *str; + uint32_t cur_len; + + while ((cur_len = pcg32_random ()) % 100000 == 0); + str = malloc (cur_len % 100000); + res = ucl_object_fromstring_common (str, cur_len % 100000, + UCL_STRING_RAW); + res->flags |= UCL_OBJECT_BINARY; + + return res; +} + +static ucl_object_t* +ucl_test_null (void) +{ + return ucl_object_typed_new (UCL_NULL); +} diff --git a/contrib/libucl/tests/test_schema.c b/contrib/libucl/tests/test_schema.c index 39eb7f399125..675764925507 100644 --- a/contrib/libucl/tests/test_schema.c +++ b/contrib/libucl/tests/test_schema.c @@ -68,9 +68,9 @@ perform_test (const ucl_object_t *schema, const ucl_object_t *obj, const ucl_object_t *valid, *data, *description; bool match; - data = ucl_object_find_key (obj, "data"); - description = ucl_object_find_key (obj, "description"); - valid = ucl_object_find_key (obj, "valid"); + data = ucl_object_lookup (obj, "data"); + description = ucl_object_lookup (obj, "description"); + valid = ucl_object_lookup (obj, "valid"); if (data == NULL || description == NULL || valid == NULL) { fprintf (stdout, "Bad test case\n"); @@ -103,9 +103,9 @@ perform_tests (const ucl_object_t *obj) return EXIT_FAILURE; } - schema = ucl_object_find_key (obj, "schema"); - tests = ucl_object_find_key (obj, "tests"); - description = ucl_object_find_key (obj, "description"); + schema = ucl_object_lookup (obj, "schema"); + tests = ucl_object_lookup (obj, "tests"); + description = ucl_object_lookup (obj, "description"); if (schema == NULL || tests == NULL || description == NULL) { fprintf (stdout, "Bad test case\n"); @@ -114,7 +114,7 @@ perform_tests (const ucl_object_t *obj) memset (&err, 0, sizeof (err)); - while ((test = ucl_iterate_object (tests, &iter, true)) != NULL) { + while ((test = ucl_object_iterate (tests, &iter, true)) != NULL) { if (!perform_test (schema, test, &err)) { fprintf (stdout, "Test suite '%s' failed\n", ucl_object_tostring (description)); @@ -151,7 +151,7 @@ main (int argc, char **argv) obj = ucl_parser_get_object (parser); ucl_parser_free (parser); - while ((elt = ucl_iterate_object (obj, &iter, true)) != NULL) { + while ((elt = ucl_object_iterate (obj, &iter, true)) != NULL) { ret = perform_tests (elt); if (ret != 0) { break; diff --git a/contrib/libucl/utils/objdump.c b/contrib/libucl/utils/objdump.c index 6fde2f4c890e..3c60c5569231 100644 --- a/contrib/libucl/utils/objdump.c +++ b/contrib/libucl/utils/objdump.c @@ -41,7 +41,7 @@ ucl_obj_dump (const ucl_object_t *obj, unsigned int shift) tmp = obj; - while ((obj = ucl_iterate_object (tmp, &it, false))) { + while ((obj = ucl_object_iterate (tmp, &it, false))) { printf ("%sucl object address: %p\n", pre + 4, obj); if (obj->key != NULL) { printf ("%skey: \"%s\"\n", pre, ucl_object_key (obj)); @@ -54,7 +54,7 @@ ucl_obj_dump (const ucl_object_t *obj, unsigned int shift) printf ("%stype: UCL_OBJECT\n", pre); printf ("%svalue: %p\n", pre, obj->value.ov); it_obj = NULL; - while ((cur = ucl_iterate_object (obj, &it_obj, true))) { + while ((cur = ucl_object_iterate (obj, &it_obj, true))) { ucl_obj_dump (cur, shift + 2); } } @@ -62,7 +62,7 @@ ucl_obj_dump (const ucl_object_t *obj, unsigned int shift) printf ("%stype: UCL_ARRAY\n", pre); printf ("%svalue: %p\n", pre, obj->value.av); it_obj = NULL; - while ((cur = ucl_iterate_object (obj, &it_obj, true))) { + while ((cur = ucl_object_iterate (obj, &it_obj, true))) { ucl_obj_dump (cur, shift + 2); } } @@ -161,7 +161,7 @@ main(int argc, char **argv) if (argc > 2) { for (k = 2; k < argc; k++) { printf ("search for \"%s\"... ", argv[k]); - par = ucl_object_find_key (obj, argv[k]); + par = ucl_object_lookup (obj, argv[k]); printf ("%sfound\n", (par == NULL )?"not ":""); ucl_obj_dump (par, 0); }