From 04726a7089e44e7ff4e6c552524841579a1053da Mon Sep 17 00:00:00 2001 From: John MacFarlane Date: Tue, 16 Jun 2015 22:21:55 -0700 Subject: Added `CMARK_OPT_VALIDATE_UTF8` option. Also command line option `--validate-utf8`. This option causes cmark to check for valid UTF-8, replacing invalid sequences with the replacement character, U+FFFD. Reinstated api tests for utf8. --- src/blocks.c | 6 +++++- src/cmark.h | 5 +++++ src/main.c | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/blocks.c b/src/blocks.c index 95a87c3..17288df 100644 --- a/src/blocks.c +++ b/src/blocks.c @@ -631,7 +631,11 @@ S_process_line(cmark_parser *parser, const unsigned char *buffer, bufsize_t byte cmark_chunk input; bool maybe_lazy; - cmark_strbuf_put(parser->curline, buffer, bytes); + if (parser->options & CMARK_OPT_VALIDATE_UTF8) { + utf8proc_check(parser->curline, buffer, bytes); + } else { + cmark_strbuf_put(parser->curline, buffer, bytes); + } parser->offset = 0; parser->column = 0; parser->blank = false; diff --git a/src/cmark.h b/src/cmark.h index d86e13e..4ff1ca0 100644 --- a/src/cmark.h +++ b/src/cmark.h @@ -506,6 +506,11 @@ char *cmark_render_commonmark(cmark_node *root, int options, int width); */ #define CMARK_OPT_SMART 8 +/** Validate UTF-8 in the input before parsing, replacing illegal + * sequences with the replacement character U+FFFD. + */ +#define CMARK_OPT_VALIDATE_UTF8 16 + /** * ## Version information */ diff --git a/src/main.c b/src/main.c index c23071f..5dc97b2 100644 --- a/src/main.c +++ b/src/main.c @@ -92,6 +92,8 @@ int main(int argc, char *argv[]) options |= CMARK_OPT_SMART; } else if (strcmp(argv[i], "--normalize") == 0) { options |= CMARK_OPT_NORMALIZE; + } else if (strcmp(argv[i], "--validate-utf8") == 0) { + options |= CMARK_OPT_VALIDATE_UTF8; } else if ((strcmp(argv[i], "--help") == 0) || (strcmp(argv[i], "-h") == 0)) { print_usage(); -- cgit v1.2.3