diff options
| author | Shawn Rutledge <shawn.rutledge@qt.io> | 2019-04-22 06:18:23 +0200 |
|---|---|---|
| committer | Shawn Rutledge <shawn.rutledge@qt.io> | 2019-05-24 15:37:05 +0200 |
| commit | 5ab6e2ef20fef198d2f1315909c756007a0c6d03 (patch) | |
| tree | 4371df06a703c715ba62484044d92ad600350e44 /src/3rdparty/md4c | |
| parent | 280d679c556ab8ead4748a627d7cd4c1950027fb (diff) | |
Update 3rdparty md4c library
0.3.3 plus a few more patches, including the one that adds
MD_BLOCK_CODE_DETAIL::fence_char, which will enable smarter rewriting
of code blocks (indented vs. fenced).
Change-Id: Ibc892369947a8a8edfa4bc20b1df98a5c8153141
Reviewed-by: Gatis Paeglis <gatis.paeglis@qt.io>
Diffstat (limited to 'src/3rdparty/md4c')
| -rw-r--r-- | src/3rdparty/md4c/md4c.c | 2319 | ||||
| -rw-r--r-- | src/3rdparty/md4c/md4c.h | 7 | ||||
| -rw-r--r-- | src/3rdparty/md4c/qt_attribution.json | 4 |
3 files changed, 1209 insertions, 1121 deletions
diff --git a/src/3rdparty/md4c/md4c.c b/src/3rdparty/md4c/md4c.c index 13c7bd3433..01e63a5fd2 100644 --- a/src/3rdparty/md4c/md4c.c +++ b/src/3rdparty/md4c/md4c.c @@ -98,6 +98,9 @@ struct MD_CTX_tag { MD_PARSER parser; void* userdata; + /* When this is true, it allows some optimizations. */ + int doc_ends_with_newline; + /* Helper temporary growing buffer. */ CHAR* buffer; unsigned alloc_buffer; @@ -124,17 +127,20 @@ struct MD_CTX_tag { #endif /* For resolving of inline spans. */ - MD_MARKCHAIN mark_chains[8]; -#define PTR_CHAIN ctx->mark_chains[0] -#define TABLECELLBOUNDARIES ctx->mark_chains[1] -#define BACKTICK_OPENERS ctx->mark_chains[2] -#define LOWERTHEN_OPENERS ctx->mark_chains[3] -#define ASTERISK_OPENERS ctx->mark_chains[4] -#define UNDERSCORE_OPENERS ctx->mark_chains[5] -#define TILDE_OPENERS ctx->mark_chains[6] -#define BRACKET_OPENERS ctx->mark_chains[7] -#define OPENERS_CHAIN_FIRST 2 -#define OPENERS_CHAIN_LAST 7 + MD_MARKCHAIN mark_chains[11]; +#define PTR_CHAIN ctx->mark_chains[0] +#define TABLECELLBOUNDARIES ctx->mark_chains[1] +#define ASTERISK_OPENERS_extraword_mod3_0 ctx->mark_chains[2] +#define ASTERISK_OPENERS_extraword_mod3_1 ctx->mark_chains[3] +#define ASTERISK_OPENERS_extraword_mod3_2 ctx->mark_chains[4] +#define ASTERISK_OPENERS_intraword_mod3_0 ctx->mark_chains[5] +#define ASTERISK_OPENERS_intraword_mod3_1 ctx->mark_chains[6] +#define ASTERISK_OPENERS_intraword_mod3_2 ctx->mark_chains[7] +#define UNDERSCORE_OPENERS ctx->mark_chains[8] +#define TILDE_OPENERS ctx->mark_chains[9] +#define BRACKET_OPENERS ctx->mark_chains[10] +#define OPENERS_CHAIN_FIRST 2 +#define OPENERS_CHAIN_LAST 10 int n_table_cell_boundaries; @@ -142,6 +148,12 @@ struct MD_CTX_tag { int unresolved_link_head; int unresolved_link_tail; + /* For resolving raw HTML. */ + OFF html_comment_horizon; + OFF html_proc_instr_horizon; + OFF html_decl_horizon; + OFF html_cdata_horizon; + /* For block analysis. * Notes: * -- It holds MD_BLOCK as well as MD_LINE structures. After each @@ -159,18 +171,16 @@ struct MD_CTX_tag { int n_containers; int alloc_containers; - int last_line_has_list_loosening_effect; - int last_list_item_starts_with_two_blank_lines; - /* Minimal indentation to call the block "indented code block". */ unsigned code_indent_offset; /* Contextual info for line analysis. */ SZ code_fence_length; /* For checking closing fence length. */ int html_block_type; /* For checking closing raw HTML condition. */ + int last_line_has_list_loosening_effect; + int last_list_item_starts_with_two_blank_lines; }; -typedef enum MD_LINETYPE_tag MD_LINETYPE; enum MD_LINETYPE_tag { MD_LINE_BLANK, MD_LINE_HR, @@ -184,6 +194,7 @@ enum MD_LINETYPE_tag { MD_LINE_TABLE, MD_LINE_TABLEUNDERLINE }; +typedef enum MD_LINETYPE_tag MD_LINETYPE; typedef struct MD_LINE_ANALYSIS_tag MD_LINE_ANALYSIS; struct MD_LINE_ANALYSIS_tag { @@ -445,209 +456,212 @@ md_text_with_null_replacement(MD_CTX* ctx, MD_TEXTTYPE type, const CHAR* str, SZ typedef struct MD_UNICODE_FOLD_INFO_tag MD_UNICODE_FOLD_INFO; struct MD_UNICODE_FOLD_INFO_tag { - int codepoints[3]; + unsigned codepoints[3]; int n_codepoints; }; #if defined MD4C_USE_UTF16 || defined MD4C_USE_UTF8 + /* Binary search over sorted "map" of codepoints. Consecutive sequences + * of codepoints may be encoded in the map by just using the + * (MIN_CODEPOINT | 0x40000000) and (MAX_CODEPOINT | 0x80000000). + * + * Returns index of the found record in the map (in the case of ranges, + * the minimal value is used); or -1 on failure. */ static int - md_is_unicode_whitespace__(int codepoint) + md_unicode_bsearch__(unsigned codepoint, const unsigned* map, size_t map_size) { - /* The ASCII ones are the most frequently used ones, so lets check them first. */ - if(codepoint <= 0x7f) - return ISWHITESPACE_(codepoint); - - /* Check for Unicode codepoints in Zs class above 127. */ - if(codepoint == 0x00a0 || codepoint == 0x1680) - return TRUE; - if(0x2000 <= codepoint && codepoint <= 0x200a) - return TRUE; - if(codepoint == 0x202f || codepoint == 0x205f || codepoint == 0x3000) - return TRUE; + int beg, end; + int pivot_beg, pivot_end; + + beg = 0; + end = (int) map_size-1; + while(beg <= end) { + /* Pivot may be a range, not just a single value. */ + pivot_beg = pivot_end = (beg + end) / 2; + if(map[pivot_end] & 0x40000000) + pivot_end++; + if(map[pivot_beg] & 0x80000000) + pivot_beg--; + + if(codepoint < (map[pivot_beg] & 0x00ffffff)) + end = pivot_beg - 1; + else if(codepoint > (map[pivot_end] & 0x00ffffff)) + beg = pivot_end + 1; + else + return pivot_beg; + } - return FALSE; + return -1; } static int - md_unicode_cmp__(const void* p_codepoint_a, const void* p_codepoint_b) + md_is_unicode_whitespace__(unsigned codepoint) { - return (*(const int*)p_codepoint_a - *(const int*)p_codepoint_b); +#define R(cp_min, cp_max) ((cp_min) | 0x40000000), ((cp_max) | 0x80000000) +#define S(cp) (cp) + /* Unicode "Zs" category. + * (generated by scripts/build_whitespace_map.py) */ + static const unsigned WHITESPACE_MAP[] = { + S(0x0020), S(0x00a0), S(0x1680), R(0x2000,0x200a), S(0x202f), S(0x205f), S(0x3000) + }; +#undef R +#undef S + + /* The ASCII ones are the most frequently used ones, also CommonMark + * specification requests few more in this range. */ + if(codepoint <= 0x7f) + return ISWHITESPACE_(codepoint); + + return (md_unicode_bsearch__(codepoint, WHITESPACE_MAP, SIZEOF_ARRAY(WHITESPACE_MAP)) >= 0); } static int - md_is_unicode_punct__(int codepoint) + md_is_unicode_punct__(unsigned codepoint) { - /* non-ASCII (above 127) Unicode punctuation codepoints (classes - * Pc, Pd, Pe, Pf, Pi, Po, Ps). - * - * Warning: Keep the array sorted. - */ - static const int punct_list[] = { - 0x00a1, 0x00a7, 0x00ab, 0x00b6, 0x00b7, 0x00bb, 0x00bf, 0x037e, 0x0387, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, 0x0589, - 0x058a, 0x05be, 0x05c0, 0x05c3, 0x05c6, 0x05f3, 0x05f4, 0x0609, 0x060a, 0x060c, 0x060d, 0x061b, 0x061e, 0x061f, 0x066a, 0x066b, - 0x066c, 0x066d, 0x06d4, 0x0700, 0x0701, 0x0702, 0x0703, 0x0704, 0x0705, 0x0706, 0x0707, 0x0708, 0x0709, 0x070a, 0x070b, 0x070c, - 0x070d, 0x07f7, 0x07f8, 0x07f9, 0x0830, 0x0831, 0x0832, 0x0833, 0x0834, 0x0835, 0x0836, 0x0837, 0x0838, 0x0839, 0x083a, 0x083b, - 0x083c, 0x083d, 0x083e, 0x085e, 0x0964, 0x0965, 0x0970, 0x0af0, 0x0df4, 0x0e4f, 0x0e5a, 0x0e5b, 0x0f04, 0x0f05, 0x0f06, 0x0f07, - 0x0f08, 0x0f09, 0x0f0a, 0x0f0b, 0x0f0c, 0x0f0d, 0x0f0e, 0x0f0f, 0x0f10, 0x0f11, 0x0f12, 0x0f14, 0x0f3a, 0x0f3b, 0x0f3c, 0x0f3d, - 0x0f85, 0x0fd0, 0x0fd1, 0x0fd2, 0x0fd3, 0x0fd4, 0x0fd9, 0x0fda, 0x104a, 0x104b, 0x104c, 0x104d, 0x104e, 0x104f, 0x10fb, 0x1360, - 0x1361, 0x1362, 0x1363, 0x1364, 0x1365, 0x1366, 0x1367, 0x1368, 0x1400, 0x166d, 0x166e, 0x169b, 0x169c, 0x16eb, 0x16ec, 0x16ed, - 0x1735, 0x1736, 0x17d4, 0x17d5, 0x17d6, 0x17d8, 0x17d9, 0x17da, 0x1800, 0x1801, 0x1802, 0x1803, 0x1804, 0x1805, 0x1806, 0x1807, - 0x1808, 0x1809, 0x180a, 0x1944, 0x1945, 0x1a1e, 0x1a1f, 0x1aa0, 0x1aa1, 0x1aa2, 0x1aa3, 0x1aa4, 0x1aa5, 0x1aa6, 0x1aa8, 0x1aa9, - 0x1aaa, 0x1aab, 0x1aac, 0x1aad, 0x1b5a, 0x1b5b, 0x1b5c, 0x1b5d, 0x1b5e, 0x1b5f, 0x1b60, 0x1bfc, 0x1bfd, 0x1bfe, 0x1bff, 0x1c3b, - 0x1c3c, 0x1c3d, 0x1c3e, 0x1c3f, 0x1c7e, 0x1c7f, 0x1cc0, 0x1cc1, 0x1cc2, 0x1cc3, 0x1cc4, 0x1cc5, 0x1cc6, 0x1cc7, 0x1cd3, 0x2010, - 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x2017, 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, 0x2020, - 0x2021, 0x2022, 0x2023, 0x2024, 0x2025, 0x2026, 0x2027, 0x2030, 0x2031, 0x2032, 0x2033, 0x2034, 0x2035, 0x2036, 0x2037, 0x2038, - 0x2039, 0x203a, 0x203b, 0x203c, 0x203d, 0x203e, 0x203f, 0x2040, 0x2041, 0x2042, 0x2043, 0x2045, 0x2046, 0x2047, 0x2048, 0x2049, - 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, 0x2050, 0x2051, 0x2053, 0x2054, 0x2055, 0x2056, 0x2057, 0x2058, 0x2059, 0x205a, - 0x205b, 0x205c, 0x205d, 0x205e, 0x207d, 0x207e, 0x208d, 0x208e, 0x2308, 0x2309, 0x230a, 0x230b, 0x2329, 0x232a, 0x2768, 0x2769, - 0x276a, 0x276b, 0x276c, 0x276d, 0x276e, 0x276f, 0x2770, 0x2771, 0x2772, 0x2773, 0x2774, 0x2775, 0x27c5, 0x27c6, 0x27e6, 0x27e7, - 0x27e8, 0x27e9, 0x27ea, 0x27eb, 0x27ec, 0x27ed, 0x27ee, 0x27ef, 0x2983, 0x2984, 0x2985, 0x2986, 0x2987, 0x2988, 0x2989, 0x298a, - 0x298b, 0x298c, 0x298d, 0x298e, 0x298f, 0x2990, 0x2991, 0x2992, 0x2993, 0x2994, 0x2995, 0x2996, 0x2997, 0x2998, 0x29d8, 0x29d9, - 0x29da, 0x29db, 0x29fc, 0x29fd, 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfe, 0x2cff, 0x2d70, 0x2e00, 0x2e01, 0x2e02, 0x2e03, 0x2e04, - 0x2e05, 0x2e06, 0x2e07, 0x2e08, 0x2e09, 0x2e0a, 0x2e0b, 0x2e0c, 0x2e0d, 0x2e0e, 0x2e0f, 0x2e10, 0x2e11, 0x2e12, 0x2e13, 0x2e14, - 0x2e15, 0x2e16, 0x2e17, 0x2e18, 0x2e19, 0x2e1a, 0x2e1b, 0x2e1c, 0x2e1d, 0x2e1e, 0x2e1f, 0x2e20, 0x2e21, 0x2e22, 0x2e23, 0x2e24, - 0x2e25, 0x2e26, 0x2e27, 0x2e28, 0x2e29, 0x2e2a, 0x2e2b, 0x2e2c, 0x2e2d, 0x2e2e, 0x2e30, 0x2e31, 0x2e32, 0x2e33, 0x2e34, 0x2e35, - 0x2e36, 0x2e37, 0x2e38, 0x2e39, 0x2e3a, 0x2e3b, 0x2e3c, 0x2e3d, 0x2e3e, 0x2e3f, 0x2e40, 0x2e41, 0x2e42, 0x2e43, 0x2e44, 0x3001, - 0x3002, 0x3003, 0x3008, 0x3009, 0x300a, 0x300b, 0x300c, 0x300d, 0x300e, 0x300f, 0x3010, 0x3011, 0x3014, 0x3015, 0x3016, 0x3017, - 0x3018, 0x3019, 0x301a, 0x301b, 0x301c, 0x301d, 0x301e, 0x301f, 0x3030, 0x303d, 0x30a0, 0x30fb, 0xa4fe, 0xa4ff, 0xa60d, 0xa60e, - 0xa60f, 0xa673, 0xa67e, 0xa6f2, 0xa6f3, 0xa6f4, 0xa6f5, 0xa6f6, 0xa6f7, 0xa874, 0xa875, 0xa876, 0xa877, 0xa8ce, 0xa8cf, 0xa8f8, - 0xa8f9, 0xa8fa, 0xa8fc, 0xa92e, 0xa92f, 0xa95f, 0xa9c1, 0xa9c2, 0xa9c3, 0xa9c4, 0xa9c5, 0xa9c6, 0xa9c7, 0xa9c8, 0xa9c9, 0xa9ca, - 0xa9cb, 0xa9cc, 0xa9cd, 0xa9de, 0xa9df, 0xaa5c, 0xaa5d, 0xaa5e, 0xaa5f, 0xaade, 0xaadf, 0xaaf0, 0xaaf1, 0xabeb, 0xfd3e, 0xfd3f, - 0xfe10, 0xfe11, 0xfe12, 0xfe13, 0xfe14, 0xfe15, 0xfe16, 0xfe17, 0xfe18, 0xfe19, 0xfe30, 0xfe31, 0xfe32, 0xfe33, 0xfe34, 0xfe35, - 0xfe36, 0xfe37, 0xfe38, 0xfe39, 0xfe3a, 0xfe3b, 0xfe3c, 0xfe3d, 0xfe3e, 0xfe3f, 0xfe40, 0xfe41, 0xfe42, 0xfe43, 0xfe44, 0xfe45, - 0xfe46, 0xfe47, 0xfe48, 0xfe49, 0xfe4a, 0xfe4b, 0xfe4c, 0xfe4d, 0xfe4e, 0xfe4f, 0xfe50, 0xfe51, 0xfe52, 0xfe54, 0xfe55, 0xfe56, - 0xfe57, 0xfe58, 0xfe59, 0xfe5a, 0xfe5b, 0xfe5c, 0xfe5d, 0xfe5e, 0xfe5f, 0xfe60, 0xfe61, 0xfe63, 0xfe68, 0xfe6a, 0xfe6b, 0xff01, - 0xff02, 0xff03, 0xff05, 0xff06, 0xff07, 0xff08, 0xff09, 0xff0a, 0xff0c, 0xff0d, 0xff0e, 0xff0f, 0xff1a, 0xff1b, 0xff1f, 0xff20, - 0xff3b, 0xff3c, 0xff3d, 0xff3f, 0xff5b, 0xff5d, 0xff5f, 0xff60, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0x10100, 0x10101, 0x10102, - 0x1039f, 0x103d0, 0x1056f, 0x10857, 0x1091f, 0x1093f, 0x10a50, 0x10a51, 0x10a52, 0x10a53, 0x10a54, 0x10a55, 0x10a56, 0x10a57, 0x10a58, 0x10a7f, - 0x10af0, 0x10af1, 0x10af2, 0x10af3, 0x10af4, 0x10af5, 0x10af6, 0x10b39, 0x10b3a, 0x10b3b, 0x10b3c, 0x10b3d, 0x10b3e, 0x10b3f, 0x10b99, 0x10b9a, - 0x10b9b, 0x10b9c, 0x11047, 0x11048, 0x11049, 0x1104a, 0x1104b, 0x1104c, 0x1104d, 0x110bb, 0x110bc, 0x110be, 0x110bf, 0x110c0, 0x110c1, 0x11140, - 0x11141, 0x11142, 0x11143, 0x11174, 0x11175, 0x111c5, 0x111c6, 0x111c7, 0x111c8, 0x111c9, 0x111cd, 0x111db, 0x111dd, 0x111de, 0x111df, 0x11238, - 0x11239, 0x1123a, 0x1123b, 0x1123c, 0x1123d, 0x112a9, 0x1144b, 0x1144c, 0x1144d, 0x1144e, 0x1144f, 0x1145b, 0x1145d, 0x114c6, 0x115c1, 0x115c2, - 0x115c3, 0x115c4, 0x115c5, 0x115c6, 0x115c7, 0x115c8, 0x115c9, 0x115ca, 0x115cb, 0x115cc, 0x115cd, 0x115ce, 0x115cf, 0x115d0, 0x115d1, 0x115d2, - 0x115d3, 0x115d4, 0x115d5, 0x115d6, 0x115d7, 0x11641, 0x11642, 0x11643, 0x11660, 0x11661, 0x11662, 0x11663, 0x11664, 0x11665, 0x11666, 0x11667, - 0x11668, 0x11669, 0x1166a, 0x1166b, 0x1166c, 0x1173c, 0x1173d, 0x1173e, 0x11c41, 0x11c42, 0x11c43, 0x11c44, 0x11c45, 0x11c70, 0x11c71, 0x12470, - 0x12471, 0x12472, 0x12473, 0x12474, 0x16a6e, 0x16a6f, 0x16af5, 0x16b37, 0x16b38, 0x16b39, 0x16b3a, 0x16b3b, 0x16b44, 0x1bc9f, 0x1da87, 0x1da88, - 0x1da89, 0x1da8a, 0x1da8b, 0x1e95e, 0x1e95f +#define R(cp_min, cp_max) ((cp_min) | 0x40000000), ((cp_max) | 0x80000000) +#define S(cp) (cp) + /* Unicode "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps" categories. + * (generated by scripts/build_punct_map.py) */ + static const unsigned PUNCT_MAP[] = { + R(0x0021,0x0023), R(0x0025,0x002a), R(0x002c,0x002f), R(0x003a,0x003b), R(0x003f,0x0040), + R(0x005b,0x005d), S(0x005f), S(0x007b), S(0x007d), S(0x00a1), S(0x00a7), S(0x00ab), R(0x00b6,0x00b7), + S(0x00bb), S(0x00bf), S(0x037e), S(0x0387), R(0x055a,0x055f), R(0x0589,0x058a), S(0x05be), S(0x05c0), + S(0x05c3), S(0x05c6), R(0x05f3,0x05f4), R(0x0609,0x060a), R(0x060c,0x060d), S(0x061b), R(0x061e,0x061f), + R(0x066a,0x066d), S(0x06d4), R(0x0700,0x070d), R(0x07f7,0x07f9), R(0x0830,0x083e), S(0x085e), + R(0x0964,0x0965), S(0x0970), S(0x09fd), S(0x0a76), S(0x0af0), S(0x0c77), S(0x0c84), S(0x0df4), S(0x0e4f), + R(0x0e5a,0x0e5b), R(0x0f04,0x0f12), S(0x0f14), R(0x0f3a,0x0f3d), S(0x0f85), R(0x0fd0,0x0fd4), + R(0x0fd9,0x0fda), R(0x104a,0x104f), S(0x10fb), R(0x1360,0x1368), S(0x1400), S(0x166e), R(0x169b,0x169c), + R(0x16eb,0x16ed), R(0x1735,0x1736), R(0x17d4,0x17d6), R(0x17d8,0x17da), R(0x1800,0x180a), + R(0x1944,0x1945), R(0x1a1e,0x1a1f), R(0x1aa0,0x1aa6), R(0x1aa8,0x1aad), R(0x1b5a,0x1b60), + R(0x1bfc,0x1bff), R(0x1c3b,0x1c3f), R(0x1c7e,0x1c7f), R(0x1cc0,0x1cc7), S(0x1cd3), R(0x2010,0x2027), + R(0x2030,0x2043), R(0x2045,0x2051), R(0x2053,0x205e), R(0x207d,0x207e), R(0x208d,0x208e), + R(0x2308,0x230b), R(0x2329,0x232a), R(0x2768,0x2775), R(0x27c5,0x27c6), R(0x27e6,0x27ef), + R(0x2983,0x2998), R(0x29d8,0x29db), R(0x29fc,0x29fd), R(0x2cf9,0x2cfc), R(0x2cfe,0x2cff), S(0x2d70), + R(0x2e00,0x2e2e), R(0x2e30,0x2e4f), R(0x3001,0x3003), R(0x3008,0x3011), R(0x3014,0x301f), S(0x3030), + S(0x303d), S(0x30a0), S(0x30fb), R(0xa4fe,0xa4ff), R(0xa60d,0xa60f), S(0xa673), S(0xa67e), + R(0xa6f2,0xa6f7), R(0xa874,0xa877), R(0xa8ce,0xa8cf), R(0xa8f8,0xa8fa), S(0xa8fc), R(0xa92e,0xa92f), + S(0xa95f), R(0xa9c1,0xa9cd), R(0xa9de,0xa9df), R(0xaa5c,0xaa5f), R(0xaade,0xaadf), R(0xaaf0,0xaaf1), + S(0xabeb), R(0xfd3e,0xfd3f), R(0xfe10,0xfe19), R(0xfe30,0xfe52), R(0xfe54,0xfe61), S(0xfe63), S(0xfe68), + R(0xfe6a,0xfe6b), R(0xff01,0xff03), R(0xff05,0xff0a), R(0xff0c,0xff0f), R(0xff1a,0xff1b), + R(0xff1f,0xff20), R(0xff3b,0xff3d), S(0xff3f), S(0xff5b), S(0xff5d), R(0xff5f,0xff65), R(0x10100,0x10102), + S(0x1039f), S(0x103d0), S(0x1056f), S(0x10857), S(0x1091f), S(0x1093f), R(0x10a50,0x10a58), S(0x10a7f), + R(0x10af0,0x10af6), R(0x10b39,0x10b3f), R(0x10b99,0x10b9c), R(0x10f55,0x10f59), R(0x11047,0x1104d), + R(0x110bb,0x110bc), R(0x110be,0x110c1), R(0x11140,0x11143), R(0x11174,0x11175), R(0x111c5,0x111c8), + S(0x111cd), S(0x111db), R(0x111dd,0x111df), R(0x11238,0x1123d), S(0x112a9), R(0x1144b,0x1144f), + S(0x1145b), S(0x1145d), S(0x114c6), R(0x115c1,0x115d7), R(0x11641,0x11643), R(0x11660,0x1166c), + R(0x1173c,0x1173e), S(0x1183b), S(0x119e2), R(0x11a3f,0x11a46), R(0x11a9a,0x11a9c), R(0x11a9e,0x11aa2), + R(0x11c41,0x11c45), R(0x11c70,0x11c71), R(0x11ef7,0x11ef8), S(0x11fff), R(0x12470,0x12474), + R(0x16a6e,0x16a6f), S(0x16af5), R(0x16b37,0x16b3b), S(0x16b44), R(0x16e97,0x16e9a), S(0x16fe2), + S(0x1bc9f), R(0x1da87,0x1da8b), R(0x1e95e,0x1e95f) }; +#undef R +#undef S - /* The ASCII ones are the most frequently used ones, so lets check them first. */ + /* The ASCII ones are the most frequently used ones, also CommonMark + * specification requests few more in this range. */ if(codepoint <= 0x7f) return ISPUNCT_(codepoint); - return (bsearch(&codepoint, punct_list, SIZEOF_ARRAY(punct_list), sizeof(int), md_unicode_cmp__) != NULL); + return (md_unicode_bsearch__(codepoint, PUNCT_MAP, SIZEOF_ARRAY(PUNCT_MAP)) >= 0); } static void - md_get_unicode_fold_info(int codepoint, MD_UNICODE_FOLD_INFO* info) + md_get_unicode_fold_info(unsigned codepoint, MD_UNICODE_FOLD_INFO* info) { - /* This maps single codepoint within a range to a single codepoint - * within an offseted range. */ - static const struct { - int min_codepoint; - int max_codepoint; - int offset; - } range_map[] = { - { 0x00c0, 0x00d6, 32 }, { 0x00d8, 0x00de, 32 }, { 0x0388, 0x038a, 37 }, { 0x0391, 0x03a1, 32 }, { 0x03a3, 0x03ab, 32 }, { 0x0400, 0x040f, 80 }, - { 0x0410, 0x042f, 32 }, { 0x0531, 0x0556, 48 }, { 0x1f08, 0x1f0f, -8 }, { 0x1f18, 0x1f1d, -8 }, { 0x1f28, 0x1f2f, -8 }, { 0x1f38, 0x1f3f, -8 }, - { 0x1f48, 0x1f4d, -8 }, { 0x1f68, 0x1f6f, -8 }, { 0x1fc8, 0x1fcb, -86 }, { 0x2160, 0x216f, 16 }, { 0x24b6, 0x24cf, 26 }, { 0xff21, 0xff3a, 32 }, - { 0x10400, 0x10425, 40 } +#define R(cp_min, cp_max) ((cp_min) | 0x40000000), ((cp_max) | 0x80000000) +#define S(cp) (cp) + /* Unicode "Pc", "Pd", "Pe", "Pf", "Pi", "Po", "Ps" categories. + * (generated by scripts/build_punct_map.py) */ + static const unsigned FOLD_MAP_1[] = { + R(0x0041,0x005a), S(0x00b5), R(0x00c0,0x00d6), R(0x00d8,0x00de), R(0x0100,0x012e), R(0x0132,0x0136), + R(0x0139,0x0147), R(0x014a,0x0176), S(0x0178), R(0x0179,0x017d), S(0x017f), S(0x0181), S(0x0182), + S(0x0186), S(0x0187), S(0x0189), S(0x018b), S(0x018e), S(0x018f), S(0x0190), S(0x0191), S(0x0193), + S(0x0194), S(0x0196), S(0x0197), S(0x0198), S(0x019c), S(0x019d), S(0x019f), R(0x01a0,0x01a4), S(0x01a6), + S(0x01a7), S(0x01a9), S(0x01ac), S(0x01ae), S(0x01af), S(0x01b1), S(0x01b3), S(0x01b7), S(0x01b8), + S(0x01bc), S(0x01c4), S(0x01c5), S(0x01c7), S(0x01c8), S(0x01ca), R(0x01cb,0x01db), R(0x01de,0x01ee), + S(0x01f1), S(0x01f2), S(0x01f6), S(0x01f7), R(0x01f8,0x021e), S(0x0220), R(0x0222,0x0232), S(0x023a), + S(0x023b), S(0x023d), S(0x023e), S(0x0241), S(0x0243), S(0x0244), S(0x0245), R(0x0246,0x024e), S(0x0345), + S(0x0370), S(0x0376), S(0x037f), S(0x0386), R(0x0388,0x038a), S(0x038c), S(0x038e), R(0x0391,0x03a1), + R(0x03a3,0x03ab), S(0x03c2), S(0x03cf), S(0x03d0), S(0x03d1), S(0x03d5), S(0x03d6), R(0x03d8,0x03ee), + S(0x03f0), S(0x03f1), S(0x03f4), S(0x03f5), S(0x03f7), S(0x03f9), S(0x03fa), R(0x03fd,0x03ff), + R(0x0400,0x040f), R(0x0410,0x042f), R(0x0460,0x0480), R(0x048a,0x04be), S(0x04c0), R(0x04c1,0x04cd), + R(0x04d0,0x052e), R(0x0531,0x0556), R(0x10a0,0x10c5), S(0x10c7), S(0x10cd), R(0x13f8,0x13fd), S(0x1c80), + S(0x1c81), S(0x1c82), S(0x1c83), S(0x1c85), S(0x1c86), S(0x1c87), S(0x1c88), R(0x1c90,0x1cba), + R(0x1cbd,0x1cbf), R(0x1e00,0x1e94), S(0x1e9b), R(0x1ea0,0x1efe), R(0x1f08,0x1f0f), R(0x1f18,0x1f1d), + R(0x1f28,0x1f2f), R(0x1f38,0x1f3f), R(0x1f48,0x1f4d), S(0x1f59), S(0x1f5b), S(0x1f5d), S(0x1f5f), + R(0x1f68,0x1f6f), S(0x1fb8), S(0x1fba), S(0x1fbe), R(0x1fc8,0x1fcb), S(0x1fd8), S(0x1fda), S(0x1fe8), + S(0x1fea), S(0x1fec), S(0x1ff8), S(0x1ffa), S(0x2126), S(0x212a), S(0x212b), S(0x2132), R(0x2160,0x216f), + S(0x2183), R(0x24b6,0x24cf), R(0x2c00,0x2c2e), S(0x2c60), S(0x2c62), S(0x2c63), S(0x2c64), + R(0x2c67,0x2c6b), S(0x2c6d), S(0x2c6e), S(0x2c6f), S(0x2c70), S(0x2c72), S(0x2c75), S(0x2c7e), + R(0x2c80,0x2ce2), S(0x2ceb), S(0x2cf2), R(0xa640,0xa66c), R(0xa680,0xa69a), R(0xa722,0xa72e), + R(0xa732,0xa76e), S(0xa779), S(0xa77d), R(0xa77e,0xa786), S(0xa78b), S(0xa78d), S(0xa790), + R(0xa796,0xa7a8), S(0xa7aa), S(0xa7ab), S(0xa7ac), S(0xa7ad), S(0xa7ae), S(0xa7b0), S(0xa7b1), S(0xa7b2), + S(0xa7b3), R(0xa7b4,0xa7be), S(0xa7c2), S(0xa7c4), S(0xa7c5), S(0xa7c6), R(0xab70,0xabbf), + R(0xff21,0xff3a), R(0x10400,0x10427), R(0x104b0,0x104d3), R(0x10c80,0x10cb2), R(0x118a0,0x118bf), + R(0x16e40,0x16e5f), R(0x1e900,0x1e921) }; - - /* This maps single codepoint to another single codepoint. */ - static const struct { - int src_codepoint; - int dest_codepoint; - } single_map[] = { - { 0x00b5, 0x03bc }, { 0x0100, 0x0101 }, { 0x0102, 0x0103 }, { 0x0104, 0x0105 }, { 0x0106, 0x0107 }, { 0x0108, 0x0109 }, { 0x010a, 0x010b }, { 0x010c, 0x010d }, - { 0x010e, 0x010f }, { 0x0110, 0x0111 }, { 0x0112, 0x0113 }, { 0x0114, 0x0115 }, { 0x0116, 0x0117 }, { 0x0118, 0x0119 }, { 0x011a, 0x011b }, { 0x011c, 0x011d }, - { 0x011e, 0x011f }, { 0x0120, 0x0121 }, { 0x0122, 0x0123 }, { 0x0124, 0x0125 }, { 0x0126, 0x0127 }, { 0x0128, 0x0129 }, { 0x012a, 0x012b }, { 0x012c, 0x012d }, - { 0x012e, 0x012f }, { 0x0132, 0x0133 }, { 0x0134, 0x0135 }, { 0x0136, 0x0137 }, { 0x0139, 0x013a }, { 0x013b, 0x013c }, { 0x013d, 0x013e }, { 0x013f, 0x0140 }, - { 0x0141, 0x0142 }, { 0x0143, 0x0144 }, { 0x0145, 0x0146 }, { 0x0147, 0x0148 }, { 0x014a, 0x014b }, { 0x014c, 0x014d }, { 0x014e, 0x014f }, { 0x0150, 0x0151 }, - { 0x0152, 0x0153 }, { 0x0154, 0x0155 }, { 0x0156, 0x0157 }, { 0x0158, 0x0159 }, { 0x015a, 0x015b }, { 0x015c, 0x015d }, { 0x015e, 0x015f }, { 0x0160, 0x0161 }, - { 0x0162, 0x0163 }, { 0x0164, 0x0165 }, { 0x0166, 0x0167 }, { 0x0168, 0x0169 }, { 0x016a, 0x016b }, { 0x016c, 0x016d }, { 0x016e, 0x016f }, { 0x0170, 0x0171 }, - { 0x0172, 0x0173 }, { 0x0174, 0x0175 }, { 0x0176, 0x0177 }, { 0x0178, 0x00ff }, { 0x0179, 0x017a }, { 0x017b, 0x017c }, { 0x017d, 0x017e }, { 0x017f, 0x0073 }, - { 0x0181, 0x0253 }, { 0x0182, 0x0183 }, { 0x0184, 0x0185 }, { 0x0186, 0x0254 }, { 0x0187, 0x0188 }, { 0x0189, 0x0256 }, { 0x018a, 0x0257 }, { 0x018b, 0x018c }, - { 0x018e, 0x01dd }, { 0x018f, 0x0259 }, { 0x0190, 0x025b }, { 0x0191, 0x0192 }, { 0x0193, 0x0260 }, { 0x0194, 0x0263 }, { 0x0196, 0x0269 }, { 0x0197, 0x0268 }, - { 0x0198, 0x0199 }, { 0x019c, 0x026f }, { 0x019d, 0x0272 }, { 0x019f, 0x0275 }, { 0x01a0, 0x01a1 }, { 0x01a2, 0x01a3 }, { 0x01a4, 0x01a5 }, { 0x01a6, 0x0280 }, - { 0x01a7, 0x01a8 }, { 0x01a9, 0x0283 }, { 0x01ac, 0x01ad }, { 0x01ae, 0x0288 }, { 0x01af, 0x01b0 }, { 0x01b1, 0x028a }, { 0x01b2, 0x028b }, { 0x01b3, 0x01b4 }, - { 0x01b5, 0x01b6 }, { 0x01b7, 0x0292 }, { 0x01b8, 0x01b9 }, { 0x01bc, 0x01bd }, { 0x01c4, 0x01c6 }, { 0x01c5, 0x01c6 }, { 0x01c7, 0x01c9 }, { 0x01c8, 0x01c9 }, - { 0x01ca, 0x01cc }, { 0x01cb, 0x01cc }, { 0x01cd, 0x01ce }, { 0x01cf, 0x01d0 }, { 0x01d1, 0x01d2 }, { 0x01d3, 0x01d4 }, { 0x01d5, 0x01d6 }, { 0x01d7, 0x01d8 }, - { 0x01d9, 0x01da }, { 0x01db, 0x01dc }, { 0x01de, 0x01df }, { 0x01e0, 0x01e1 }, { 0x01e2, 0x01e3 }, { 0x01e4, 0x01e5 }, { 0x01e6, 0x01e7 }, { 0x01e8, 0x01e9 }, - { 0x01ea, 0x01eb }, { 0x01ec, 0x01ed }, { 0x01ee, 0x01ef }, { 0x01f1, 0x01f3 }, { 0x01f2, 0x01f3 }, { 0x01f4, 0x01f5 }, { 0x01f6, 0x0195 }, { 0x01f7, 0x01bf }, - { 0x01f8, 0x01f9 }, { 0x01fa, 0x01fb }, { 0x01fc, 0x01fd }, { 0x01fe, 0x01ff }, { 0x0200, 0x0201 }, { 0x0202, 0x0203 }, { 0x0204, 0x0205 }, { 0x0206, 0x0207 }, - { 0x0208, 0x0209 }, { 0x020a, 0x020b }, { 0x020c, 0x020d }, { 0x020e, 0x020f }, { 0x0210, 0x0211 }, { 0x0212, 0x0213 }, { 0x0214, 0x0215 }, { 0x0216, 0x0217 }, - { 0x0218, 0x0219 }, { 0x021a, 0x021b }, { 0x021c, 0x021d }, { 0x021e, 0x021f }, { 0x0220, 0x019e }, { 0x0222, 0x0223 }, { 0x0224, 0x0225 }, { 0x0226, 0x0227 }, - { 0x0228, 0x0229 }, { 0x022a, 0x022b }, { 0x022c, 0x022d }, { 0x022e, 0x022f }, { 0x0230, 0x0231 }, { 0x0232, 0x0233 }, { 0x0345, 0x03b9 }, { 0x0386, 0x03ac }, - { 0x038c, 0x03cc }, { 0x038e, 0x03cd }, { 0x038f, 0x03ce }, { 0x03c2, 0x03c3 }, { 0x03d0, 0x03b2 }, { 0x03d1, 0x03b8 }, { 0x03d5, 0x03c6 }, { 0x03d6, 0x03c0 }, - { 0x03d8, 0x03d9 }, { 0x03da, 0x03db }, { 0x03dc, 0x03dd }, { 0x03de, 0x03df }, { 0x03e0, 0x03e1 }, { 0x03e2, 0x03e3 }, { 0x03e4, 0x03e5 }, { 0x03e6, 0x03e7 }, - { 0x03e8, 0x03e9 }, { 0x03ea, 0x03eb }, { 0x03ec, 0x03ed }, { 0x03ee, 0x03ef }, { 0x03f0, 0x03ba }, { 0x03f1, 0x03c1 }, { 0x03f2, 0x03c3 }, { 0x03f4, 0x03b8 }, - { 0x03f5, 0x03b5 }, { 0x0460, 0x0461 }, { 0x0462, 0x0463 }, { 0x0464, 0x0465 }, { 0x0466, 0x0467 }, { 0x0468, 0x0469 }, { 0x046a, 0x046b }, { 0x046c, 0x046d }, - { 0x046e, 0x046f }, { 0x0470, 0x0471 }, { 0x0472, 0x0473 }, { 0x0474, 0x0475 }, { 0x0476, 0x0477 }, { 0x0478, 0x0479 }, { 0x047a, 0x047b }, { 0x047c, 0x047d }, - { 0x047e, 0x047f }, { 0x0480, 0x0481 }, { 0x048a, 0x048b }, { 0x048c, 0x048d }, { 0x048e, 0x048f }, { 0x0490, 0x0491 }, { 0x0492, 0x0493 }, { 0x0494, 0x0495 }, - { 0x0496, 0x0497 }, { 0x0498, 0x0499 }, { 0x049a, 0x049b }, { 0x049c, 0x049d }, { 0x049e, 0x049f }, { 0x04a0, 0x04a1 }, { 0x04a2, 0x04a3 }, { 0x04a4, 0x04a5 }, - { 0x04a6, 0x04a7 }, { 0x04a8, 0x04a9 }, { 0x04aa, 0x04ab }, { 0x04ac, 0x04ad }, { 0x04ae, 0x04af }, { 0x04b0, 0x04b1 }, { 0x04b2, 0x04b3 }, { 0x04b4, 0x04b5 }, - { 0x04b6, 0x04b7 }, { 0x04b8, 0x04b9 }, { 0x04ba, 0x04bb }, { 0x04bc, 0x04bd }, { 0x04be, 0x04bf }, { 0x04c1, 0x04c2 }, { 0x04c3, 0x04c4 }, { 0x04c5, 0x04c6 }, - { 0x04c7, 0x04c8 }, { 0x04c9, 0x04ca }, { 0x04cb, 0x04cc }, { 0x04cd, 0x04ce }, { 0x04d0, 0x04d1 }, { 0x04d2, 0x04d3 }, { 0x04d4, 0x04d5 }, { 0x04d6, 0x04d7 }, - { 0x04d8, 0x04d9 }, { 0x04da, 0x04db }, { 0x04dc, 0x04dd }, { 0x04de, 0x04df }, { 0x04e0, 0x04e1 }, { 0x04e2, 0x04e3 }, { 0x04e4, 0x04e5 }, { 0x04e6, 0x04e7 }, - { 0x04e8, 0x04e9 }, { 0x04ea, 0x04eb }, { 0x04ec, 0x04ed }, { 0x04ee, 0x04ef }, { 0x04f0, 0x04f1 }, { 0x04f2, 0x04f3 }, { 0x04f4, 0x04f5 }, { 0x04f8, 0x04f9 }, - { 0x0500, 0x0501 }, { 0x0502, 0x0503 }, { 0x0504, 0x0505 }, { 0x0506, 0x0507 }, { 0x0508, 0x0509 }, { 0x050a, 0x050b }, { 0x050c, 0x050d }, { 0x050e, 0x050f }, - { 0x1e00, 0x1e01 }, { 0x1e02, 0x1e03 }, { 0x1e04, 0x1e05 }, { 0x1e06, 0x1e07 }, { 0x1e08, 0x1e09 }, { 0x1e0a, 0x1e0b }, { 0x1e0c, 0x1e0d }, { 0x1e0e, 0x1e0f }, - { 0x1e10, 0x1e11 }, { 0x1e12, 0x1e13 }, { 0x1e14, 0x1e15 }, { 0x1e16, 0x1e17 }, { 0x1e18, 0x1e19 }, { 0x1e1a, 0x1e1b }, { 0x1e1c, 0x1e1d }, { 0x1e1e, 0x1e1f }, - { 0x1e20, 0x1e21 }, { 0x1e22, 0x1e23 }, { 0x1e24, 0x1e25 }, { 0x1e26, 0x1e27 }, { 0x1e28, 0x1e29 }, { 0x1e2a, 0x1e2b }, { 0x1e2c, 0x1e2d }, { 0x1e2e, 0x1e2f }, - { 0x1e30, 0x1e31 }, { 0x1e32, 0x1e33 }, { 0x1e34, 0x1e35 }, { 0x1e36, 0x1e37 }, { 0x1e38, 0x1e39 }, { 0x1e3a, 0x1e3b }, { 0x1e3c, 0x1e3d }, { 0x1e3e, 0x1e3f }, - { 0x1e40, 0x1e41 }, { 0x1e42, 0x1e43 }, { 0x1e44, 0x1e45 }, { 0x1e46, 0x1e47 }, { 0x1e48, 0x1e49 }, { 0x1e4a, 0x1e4b }, { 0x1e4c, 0x1e4d }, { 0x1e4e, 0x1e4f }, - { 0x1e50, 0x1e51 }, { 0x1e52, 0x1e53 }, { 0x1e54, 0x1e55 }, { 0x1e56, 0x1e57 }, { 0x1e58, 0x1e59 }, { 0x1e5a, 0x1e5b }, { 0x1e5c, 0x1e5d }, { 0x1e5e, 0x1e5f }, - { 0x1e60, 0x1e61 }, { 0x1e62, 0x1e63 }, { 0x1e64, 0x1e65 }, { 0x1e66, 0x1e67 }, { 0x1e68, 0x1e69 }, { 0x1e6a, 0x1e6b }, { 0x1e6c, 0x1e6d }, { 0x1e6e, 0x1e6f }, - { 0x1e70, 0x1e71 }, { 0x1e72, 0x1e73 }, { 0x1e74, 0x1e75 }, { 0x1e76, 0x1e77 }, { 0x1e78, 0x1e79 }, { 0x1e7a, 0x1e7b }, { 0x1e7c, 0x1e7d }, { 0x1e7e, 0x1e7f }, - { 0x1e80, 0x1e81 }, { 0x1e82, 0x1e83 }, { 0x1e84, 0x1e85 }, { 0x1e86, 0x1e87 }, { 0x1e88, 0x1e89 }, { 0x1e8a, 0x1e8b }, { 0x1e8c, 0x1e8d }, { 0x1e8e, 0x1e8f }, - { 0x1e90, 0x1e91 }, { 0x1e92, 0x1e93 }, { 0x1e94, 0x1e95 }, { 0x1e9b, 0x1e61 }, { 0x1ea0, 0x1ea1 }, { 0x1ea2, 0x1ea3 }, { 0x1ea4, 0x1ea5 }, { 0x1ea6, 0x1ea7 }, - { 0x1ea8, 0x1ea9 }, { 0x1eaa, 0x1eab }, { 0x1eac, 0x1ead }, { 0x1eae, 0x1eaf }, { 0x1eb0, 0x1eb1 }, { 0x1eb2, 0x1eb3 }, { 0x1eb4, 0x1eb5 }, { 0x1eb6, 0x1eb7 }, - { 0x1eb8, 0x1eb9 }, { 0x1eba, 0x1ebb }, { 0x1ebc, 0x1ebd }, { 0x1ebe, 0x1ebf }, { 0x1ec0, 0x1ec1 }, { 0x1ec2, 0x1ec3 }, { 0x1ec4, 0x1ec5 }, { 0x1ec6, 0x1ec7 }, - { 0x1ec8, 0x1ec9 }, { 0x1eca, 0x1ecb }, { 0x1ecc, 0x1ecd }, { 0x1ece, 0x1ecf }, { 0x1ed0, 0x1ed1 }, { 0x1ed2, 0x1ed3 }, { 0x1ed4, 0x1ed5 }, { 0x1ed6, 0x1ed7 }, - { 0x1ed8, 0x1ed9 }, { 0x1eda, 0x1edb }, { 0x1edc, 0x1edd }, { 0x1ede, 0x1edf }, { 0x1ee0, 0x1ee1 }, { 0x1ee2, 0x1ee3 }, { 0x1ee4, 0x1ee5 }, { 0x1ee6, 0x1ee7 }, - { 0x1ee8, 0x1ee9 }, { 0x1eea, 0x1eeb }, { 0x1eec, 0x1eed }, { 0x1eee, 0x1eef }, { 0x1ef0, 0x1ef1 }, { 0x1ef2, 0x1ef3 }, { 0x1ef4, 0x1ef5 }, { 0x1ef6, 0x1ef7 }, - { 0x1ef8, 0x1ef9 }, { 0x1f59, 0x1f51 }, { 0x1f5b, 0x1f53 }, { 0x1f5d, 0x1f55 }, { 0x1f5f, 0x1f57 }, { 0x1fb8, 0x1fb0 }, { 0x1fb9, 0x1fb1 }, { 0x1fba, 0x1f70 }, - { 0x1fbb, 0x1f71 }, { 0x1fbe, 0x03b9 }, { 0x1fd8, 0x1fd0 }, { 0x1fd9, 0x1fd1 }, { 0x1fda, 0x1f76 }, { 0x1fdb, 0x1f77 }, { 0x1fe8, 0x1fe0 }, { 0x1fe9, 0x1fe1 }, - { 0x1fea, 0x1f7a }, { 0x1feb, 0x1f7b }, { 0x1fec, 0x1fe5 }, { 0x1ff8, 0x1f78 }, { 0x1ff9, 0x1f79 }, { 0x1ffa, 0x1f7c }, { 0x1ffb, 0x1f7d }, { 0x2126, 0x03c9 }, - { 0x212a, 0x006b }, { 0x212b, 0x00e5 }, + static const unsigned FOLD_MAP_1_DATA[] = { + 0x0061, 0x007a, 0x03bc, 0x00e0, 0x00f6, 0x00f8, 0x00fe, 0x0101, 0x012f, 0x0133, 0x0137, 0x013a, 0x0148, + 0x014b, 0x0177, 0x00ff, 0x017a, 0x017e, 0x0073, 0x0253, 0x0183, 0x0254, 0x0188, 0x0256, 0x018c, 0x01dd, + 0x0259, 0x025b, 0x0192, 0x0260, 0x0263, 0x0269, 0x0268, 0x0199, 0x026f, 0x0272, 0x0275, 0x01a1, 0x01a5, + 0x0280, 0x01a8, 0x0283, 0x01ad, 0x0288, 0x01b0, 0x028a, 0x01b4, 0x0292, 0x01b9, 0x01bd, 0x01c6, 0x01c6, + 0x01c9, 0x01c9, 0x01cc, 0x01cc, 0x01dc, 0x01df, 0x01ef, 0x01f3, 0x01f3, 0x0195, 0x01bf, 0x01f9, 0x021f, + 0x019e, 0x0223, 0x0233, 0x2c65, 0x023c, 0x019a, 0x2c66, 0x0242, 0x0180, 0x0289, 0x028c, 0x0247, 0x024f, + 0x03b9, 0x0371, 0x0377, 0x03f3, 0x03ac, 0x03ad, 0x03af, 0x03cc, 0x03cd, 0x03b1, 0x03c1, 0x03c3, 0x03cb, + 0x03c3, 0x03d7, 0x03b2, 0x03b8, 0x03c6, 0x03c0, 0x03d9, 0x03ef, 0x03ba, 0x03c1, 0x03b8, 0x03b5, 0x03f8, + 0x03f2, 0x03fb, 0x037b, 0x037d, 0x0450, 0x045f, 0x0430, 0x044f, 0x0461, 0x0481, 0x048b, 0x04bf, 0x04cf, + 0x04c2, 0x04ce, 0x04d1, 0x052f, 0x0561, 0x0586, 0x2d00, 0x2d25, 0x2d27, 0x2d2d, 0x13f0, 0x13f5, 0x0432, + 0x0434, 0x043e, 0x0441, 0x0442, 0x044a, 0x0463, 0xa64b, 0x10d0, 0x10fa, 0x10fd, 0x10ff, 0x1e01, 0x1e95, + 0x1e61, 0x1ea1, 0x1eff, 0x1f00, 0x1f07, 0x1f10, 0x1f15, 0x1f20, 0x1f27, 0x1f30, 0x1f37, 0x1f40, 0x1f45, + 0x1f51, 0x1f53, 0x1f55, 0x1f57, 0x1f60, 0x1f67, 0x1fb0, 0x1f70, 0x03b9, 0x1f72, 0x1f75, 0x1fd0, 0x1f76, + 0x1fe0, 0x1f7a, 0x1fe5, 0x1f78, 0x1f7c, 0x03c9, 0x006b, 0x00e5, 0x214e, 0x2170, 0x217f, 0x2184, 0x24d0, + 0x24e9, 0x2c30, 0x2c5e, 0x2c61, 0x026b, 0x1d7d, 0x027d, 0x2c68, 0x2c6c, 0x0251, 0x0271, 0x0250, 0x0252, + 0x2c73, 0x2c76, 0x023f, 0x2c81, 0x2ce3, 0x2cec, 0x2cf3, 0xa641, 0xa66d, 0xa681, 0xa69b, 0xa723, 0xa72f, + 0xa733, 0xa76f, 0xa77a, 0x1d79, 0xa77f, 0xa787, 0xa78c, 0x0265, 0xa791, 0xa797, 0xa7a9, 0x0266, 0x025c, + 0x0261, 0x026c, 0x026a, 0x029e, 0x0287, 0x029d, 0xab53, 0xa7b5, 0xa7bf, 0xa7c3, 0xa794, 0x0282, 0x1d8e, + 0x13a0, 0x13ef, 0xff41, 0xff5a, 0x10428, 0x1044f, 0x104d8, 0x104fb, 0x10cc0, 0x10cf2, 0x118c0, 0x118df, + 0x16e60, 0x16e7f, 0x1e922, 0x1e943 }; - - /* This maps single codepoint to two codepoints. */ - static const struct { - int src_codepoint; - int dest_codepoint0; - int dest_codepoint1; - } double_map[] = { - { 0x00df, 0x0073, 0x0073 }, { 0x0130, 0x0069, 0x0307 }, { 0x0149, 0x02bc, 0x006e }, { 0x01f0, 0x006a, 0x030c }, { 0x0587, 0x0565, 0x0582 }, { 0x1e96, 0x0068, 0x0331 }, - { 0x1e97, 0x0074, 0x0308 }, { 0x1e98, 0x0077, 0x030a }, { 0x1e99, 0x0079, 0x030a }, { 0x1e9a, 0x0061, 0x02be }, { 0x1f50, 0x03c5, 0x0313 }, { 0x1f80, 0x1f00, 0x03b9 }, - { 0x1f81, 0x1f01, 0x03b9 }, { 0x1f82, 0x1f02, 0x03b9 }, { 0x1f83, 0x1f03, 0x03b9 }, { 0x1f84, 0x1f04, 0x03b9 }, { 0x1f85, 0x1f05, 0x03b9 }, { 0x1f86, 0x1f06, 0x03b9 }, - { 0x1f87, 0x1f07, 0x03b9 }, { 0x1f88, 0x1f00, 0x03b9 }, { 0x1f89, 0x1f01, 0x03b9 }, { 0x1f8a, 0x1f02, 0x03b9 }, { 0x1f8b, 0x1f03, 0x03b9 }, { 0x1f8c, 0x1f04, 0x03b9 }, - { 0x1f8d, 0x1f05, 0x03b9 }, { 0x1f8e, 0x1f06, 0x03b9 }, { 0x1f8f, 0x1f07, 0x03b9 }, { 0x1f90, 0x1f20, 0x03b9 }, { 0x1f91, 0x1f21, 0x03b9 }, { 0x1f92, 0x1f22, 0x03b9 }, - { 0x1f93, 0x1f23, 0x03b9 }, { 0x1f94, 0x1f24, 0x03b9 }, { 0x1f95, 0x1f25, 0x03b9 }, { 0x1f96, 0x1f26, 0x03b9 }, { 0x1f97, 0x1f27, 0x03b9 }, { 0x1f98, 0x1f20, 0x03b9 }, - { 0x1f99, 0x1f21, 0x03b9 }, { 0x1f9a, 0x1f22, 0x03b9 }, { 0x1f9b, 0x1f23, 0x03b9 }, { 0x1f9c, 0x1f24, 0x03b9 }, { 0x1f9d, 0x1f25, 0x03b9 }, { 0x1f9e, 0x1f26, 0x03b9 }, - { 0x1f9f, 0x1f27, 0x03b9 }, { 0x1fa0, 0x1f60, 0x03b9 }, { 0x1fa1, 0x1f61, 0x03b9 }, { 0x1fa2, 0x1f62, 0x03b9 }, { 0x1fa3, 0x1f63, 0x03b9 }, { 0x1fa4, 0x1f64, 0x03b9 }, - { 0x1fa5, 0x1f65, 0x03b9 }, { 0x1fa6, 0x1f66, 0x03b9 }, { 0x1fa7, 0x1f67, 0x03b9 }, { 0x1fa8, 0x1f60, 0x03b9 }, { 0x1fa9, 0x1f61, 0x03b9 }, { 0x1faa, 0x1f62, 0x03b9 }, - { 0x1fab, 0x1f63, 0x03b9 }, { 0x1fac, 0x1f64, 0x03b9 }, { 0x1fad, 0x1f65, 0x03b9 }, { 0x1fae, 0x1f66, 0x03b9 }, { 0x1faf, 0x1f67, 0x03b9 }, { 0x1fb2, 0x1f70, 0x03b9 }, - { 0x1fb3, 0x03b1, 0x03b9 }, { 0x1fb4, 0x03ac, 0x03b9 }, { 0x1fb6, 0x03b1, 0x0342 }, { 0x1fbc, 0x03b1, 0x03b9 }, { 0x1fc2, 0x1f74, 0x03b9 }, { 0x1fc3, 0x03b7, 0x03b9 }, - { 0x1fc4, 0x03ae, 0x03b9 }, { 0x1fc6, 0x03b7, 0x0342 }, { 0x1fcc, 0x03b7, 0x03b9 }, { 0x1fd6, 0x03b9, 0x0342 }, { 0x1fe4, 0x03c1, 0x0313 }, { 0x1fe6, 0x03c5, 0x0342 }, - { 0x1ff2, 0x1f7c, 0x03b9 }, { 0x1ff3, 0x03c9, 0x03b9 }, { 0x1ff4, 0x03ce, 0x03b9 }, { 0x1ff6, 0x03c9, 0x0342 }, { 0x1ffc, 0x03c9, 0x03b9 }, { 0xfb00, 0x0066, 0x0066 }, - { 0xfb01, 0x0066, 0x0069 }, { 0xfb02, 0x0066, 0x006c }, { 0xfb05, 0x0073, 0x0074 }, { 0xfb06, 0x0073, 0x0074 }, { 0xfb13, 0x0574, 0x0576 }, { 0xfb14, 0x0574, 0x0565 }, - { 0xfb15, 0x0574, 0x056b }, { 0xfb16, 0x057e, 0x0576 }, { 0xfb17, 0x0574, 0x056d } + static const unsigned FOLD_MAP_2[] = { + S(0x00df), S(0x0130), S(0x0149), S(0x01f0), S(0x0587), S(0x1e96), S(0x1e97), S(0x1e98), S(0x1e99), + S(0x1e9a), S(0x1e9e), S(0x1f50), R(0x1f80,0x1f87), R(0x1f88,0x1f8f), R(0x1f90,0x1f97), R(0x1f98,0x1f9f), + R(0x1fa0,0x1fa7), R(0x1fa8,0x1faf), S(0x1fb2), S(0x1fb3), S(0x1fb4), S(0x1fb6), S(0x1fbc), S(0x1fc2), + S(0x1fc3), S(0x1fc4), S(0x1fc6), S(0x1fcc), S(0x1fd6), S(0x1fe4), S(0x1fe6), S(0x1ff2), S(0x1ff3), + S(0x1ff4), S(0x1ff6), S(0x1ffc), S(0xfb00), S(0xfb01), S(0xfb02), S(0xfb05), S(0xfb06), S(0xfb13), + S(0xfb14), S(0xfb15), S(0xfb16), S(0xfb17) }; - - /* This maps single codepoint to three codepoints. */ + static const unsigned FOLD_MAP_2_DATA[] = { + 0x0073,0x0073, 0x0069,0x0307, 0x02bc,0x006e, 0x006a,0x030c, 0x0565,0x0582, 0x0068,0x0331, 0x0074,0x0308, + 0x0077,0x030a, 0x0079,0x030a, 0x0061,0x02be, 0x0073,0x0073, 0x03c5,0x0313, 0x1f00,0x03b9, 0x1f07,0x03b9, + 0x1f00,0x03b9, 0x1f07,0x03b9, 0x1f20,0x03b9, 0x1f27,0x03b9, 0x1f20,0x03b9, 0x1f27,0x03b9, 0x1f60,0x03b9, + 0x1f67,0x03b9, 0x1f60,0x03b9, 0x1f67,0x03b9, 0x1f70,0x03b9, 0x03b1,0x03b9, 0x03ac,0x03b9, 0x03b1,0x0342, + 0x03b1,0x03b9, 0x1f74,0x03b9, 0x03b7,0x03b9, 0x03ae,0x03b9, 0x03b7,0x0342, 0x03b7,0x03b9, 0x03b9,0x0342, + 0x03c1,0x0313, 0x03c5,0x0342, 0x1f7c,0x03b9, 0x03c9,0x03b9, 0x03ce,0x03b9, 0x03c9,0x0342, 0x03c9,0x03b9, + 0x0066,0x0066, 0x0066,0x0069, 0x0066,0x006c, 0x0073,0x0074, 0x0073,0x0074, 0x0574,0x0576, 0x0574,0x0565, + 0x0574,0x056b, 0x057e,0x0576, 0x0574,0x056d + }; + static const unsigned FOLD_MAP_3[] = { + S(0x0390), S(0x03b0), S(0x1f52), S(0x1f54), S(0x1f56), S(0x1fb7), S(0x1fc7), S(0x1fd2), S(0x1fd3), + S(0x1fd7), S(0x1fe2), S(0x1fe3), S(0x1fe7), S(0x1ff7), S(0xfb03), S(0xfb04) + }; + static const unsigned FOLD_MAP_3_DATA[] = { + 0x03b9,0x0308,0x0301, 0x03c5,0x0308,0x0301, 0x03c5,0x0313,0x0300, 0x03c5,0x0313,0x0301, + 0x03c5,0x0313,0x0342, 0x03b1,0x0342,0x03b9, 0x03b7,0x0342,0x03b9, 0x03b9,0x0308,0x0300, + 0x03b9,0x0308,0x0301, 0x03b9,0x0308,0x0342, 0x03c5,0x0308,0x0300, 0x03c5,0x0308,0x0301, + 0x03c5,0x0308,0x0342, 0x03c9,0x0342,0x03b9, 0x0066,0x0066,0x0069, 0x0066,0x0066,0x006c + }; +#undef R +#undef S static const struct { - int src_codepoint; - int dest_codepoint0; - int dest_codepoint1; - int dest_codepoint2; - } triple_map[] = { - { 0x0390, 0x03b9, 0x0308, 0x0301 }, { 0x03b0, 0x03c5, 0x0308, 0x0301 }, { 0x1f52, 0x03c5, 0x0313, 0x0300 }, { 0x1f54, 0x03c5, 0x0313, 0x0301 }, - { 0x1f56, 0x03c5, 0x0313, 0x0342 }, { 0x1fb7, 0x03b1, 0x0342, 0x03b9 }, { 0x1fc7, 0x03b7, 0x0342, 0x03b9 }, { 0x1fd2, 0x03b9, 0x0308, 0x0300 }, - { 0x1fd3, 0x03b9, 0x0308, 0x0301 }, { 0x1fd7, 0x03b9, 0x0308, 0x0342 }, { 0x1fe2, 0x03c5, 0x0308, 0x0300 }, { 0x1fe3, 0x03c5, 0x0308, 0x0301 }, - { 0x1fe7, 0x03c5, 0x0308, 0x0342 }, { 0x1ff7, 0x03c9, 0x0342, 0x03b9 }, { 0xfb03, 0x0066, 0x0066, 0x0069 }, { 0xfb04, 0x0066, 0x0066, 0x006c } + const unsigned* map; + const unsigned* data; + size_t map_size; + int n_codepoints; + } FOLD_MAP_LIST[] = { + { FOLD_MAP_1, FOLD_MAP_1_DATA, SIZEOF_ARRAY(FOLD_MAP_1), 1 }, + { FOLD_MAP_2, FOLD_MAP_2_DATA, SIZEOF_ARRAY(FOLD_MAP_2), 2 }, + { FOLD_MAP_3, FOLD_MAP_3_DATA, SIZEOF_ARRAY(FOLD_MAP_3), 3 } }; int i; @@ -661,41 +675,37 @@ struct MD_UNICODE_FOLD_INFO_tag { return; } - for(i = 0; i < SIZEOF_ARRAY(range_map); i++) { - if(range_map[i].min_codepoint <= codepoint && codepoint <= range_map[i].max_codepoint) { - info->codepoints[0] = codepoint + range_map[i].offset; - info->n_codepoints = 1; - return; - } - } - - for(i = 0; i < SIZEOF_ARRAY(single_map); i++) { - if(codepoint == single_map[i].src_codepoint) { - info->codepoints[0] = single_map[i].dest_codepoint; - info->n_codepoints = 1; - return; - } - } - - for(i = 0; i < SIZEOF_ARRAY(double_map); i++) { - if(codepoint == double_map[i].src_codepoint) { - info->codepoints[0] = double_map[i].dest_codepoint0; - info->codepoints[1] = double_map[i].dest_codepoint1; - info->n_codepoints = 2; - return; - } - } + /* Try to locate the codepoint in any of the maps. */ + for(i = 0; i < (int) SIZEOF_ARRAY(FOLD_MAP_LIST); i++) { + int index; + + index = md_unicode_bsearch__(codepoint, FOLD_MAP_LIST[i].map, FOLD_MAP_LIST[i].map_size); + if(index >= 0) { + /* Found the mapping. */ + int n_codepoints = FOLD_MAP_LIST[i].n_codepoints; + const unsigned* map = FOLD_MAP_LIST[i].map; + const unsigned* codepoints = FOLD_MAP_LIST[i].data + (index * n_codepoints); + + memcpy(info->codepoints, codepoints, sizeof(unsigned) * n_codepoints); + info->n_codepoints = n_codepoints; + + if(FOLD_MAP_LIST[i].map[index] != codepoint) { + /* The found mapping maps whole range of codepoints, + * i.e. we have to offset info->codepoints[0] accordingly. */ + if((map[index] & 0x00ffffff)+1 == codepoints[0]) { + /* Alternating type of the range. */ + info->codepoints[0] = codepoint + ((codepoint & 0x1) == (map[index] & 0x1) ? 1 : 0); + } else { + /* Range to range kind of mapping. */ + info->codepoints[0] += (codepoint - (map[index] & 0x00ffffff)); + } + } - for(i = 0; i < SIZEOF_ARRAY(triple_map); i++) { - if(codepoint == triple_map[i].src_codepoint) { - info->codepoints[0] = triple_map[i].dest_codepoint0; - info->codepoints[1] = triple_map[i].dest_codepoint1; - info->codepoints[2] = triple_map[i].dest_codepoint2; - info->n_codepoints = 3; return; } } + /* No mapping found. Map the codepoint to itself. */ info->codepoints[0] = codepoint; info->n_codepoints = 1; } @@ -707,7 +717,7 @@ struct MD_UNICODE_FOLD_INFO_tag { #define IS_UTF16_SURROGATE_LO(word) (((WORD)(word) & 0xfc00) == 0xdc00) #define UTF16_DECODE_SURROGATE(hi, lo) (0x10000 + ((((unsigned)(hi) & 0x3ff) << 10) | (((unsigned)(lo) & 0x3ff) << 0))) - static int + static unsigned md_decode_utf16le__(const CHAR* str, SZ str_size, SZ* p_size) { if(IS_UTF16_SURROGATE_HI(str[0])) { @@ -723,7 +733,7 @@ struct MD_UNICODE_FOLD_INFO_tag { return str[0]; } - static int + static unsigned md_decode_utf16le_before__(MD_CTX* ctx, OFF off) { if(off > 2 && IS_UTF16_SURROGATE_HI(CH(off-2)) && IS_UTF16_SURROGATE_LO(CH(off-1))) @@ -752,7 +762,7 @@ struct MD_UNICODE_FOLD_INFO_tag { #define IS_UTF8_LEAD4(byte) (((unsigned char)(byte) & 0xf8) == 0xf0) #define IS_UTF8_TAIL(byte) (((unsigned char)(byte) & 0xc0) == 0x80) - static int + static unsigned md_decode_utf8__(const CHAR* str, SZ str_size, SZ* p_size) { if(!IS_UTF8_LEAD1(str[0])) { @@ -788,10 +798,10 @@ struct MD_UNICODE_FOLD_INFO_tag { if(p_size != NULL) *p_size = 1; - return str[0]; + return (unsigned) str[0]; } - static int + static unsigned md_decode_utf8_before__(MD_CTX* ctx, OFF off) { if(!IS_UTF8_LEAD1(CH(off-1))) { @@ -811,7 +821,7 @@ struct MD_UNICODE_FOLD_INFO_tag { (((unsigned int)CH(off-1) & 0x3f) << 0); } - return CH(off-1); + return (unsigned) CH(off-1); } #define ISUNICODEWHITESPACE_(codepoint) md_is_unicode_whitespace__(codepoint) @@ -821,7 +831,7 @@ struct MD_UNICODE_FOLD_INFO_tag { #define ISUNICODEPUNCT(off) md_is_unicode_punct__(md_decode_utf8__(STR(off), ctx->size - (off), NULL)) #define ISUNICODEPUNCTBEFORE(off) md_is_unicode_punct__(md_decode_utf8_before__(ctx, off)) - static inline int + static inline unsigned md_decode_unicode(const CHAR* str, OFF off, SZ str_size, SZ* p_char_size) { return md_decode_utf8__(str+off, str_size-off, p_char_size); @@ -835,7 +845,7 @@ struct MD_UNICODE_FOLD_INFO_tag { #define ISUNICODEPUNCTBEFORE(off) ISPUNCT((off)-1) static inline void - md_get_unicode_fold_info(int codepoint, MD_UNICODE_FOLD_INFO* info) + md_get_unicode_fold_info(unsigned codepoint, MD_UNICODE_FOLD_INFO* info) { info->codepoints[0] = codepoint; if(ISUPPER_(codepoint)) @@ -843,11 +853,11 @@ struct MD_UNICODE_FOLD_INFO_tag { info->n_codepoints = 1; } - static inline int + static inline unsigned md_decode_unicode(const CHAR* str, OFF off, SZ str_size, SZ* p_size) { *p_size = 1; - return str[off]; + return (unsigned) str[off]; } #endif @@ -921,7 +931,7 @@ static OFF md_skip_unicode_whitespace(const CHAR* label, OFF off, SZ size) { SZ char_size; - int codepoint; + unsigned codepoint; while(off < size) { codepoint = md_decode_unicode(label, off, size, &char_size); @@ -1064,108 +1074,93 @@ done: } static int -md_is_html_comment(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) +md_scan_for_html_closer(MD_CTX* ctx, const MD_CHAR* str, MD_SIZE len, + const MD_LINE* lines, int n_lines, + OFF beg, OFF max_end, OFF* p_end, + OFF* p_scan_horizon) { OFF off = beg; int i = 0; - MD_ASSERT(CH(beg) == _T('<')); - - if(off + 4 >= lines[0].end) - return FALSE; - if(CH(off+1) != _T('!') || CH(off+2) != _T('-') || CH(off+3) != _T('-')) - return FALSE; - off += 4; - - /* ">" and "->" must follow the opening. */ - if(off < lines[0].end && CH(off) == _T('>')) - return FALSE; - if(off+1 < lines[0].end && CH(off) == _T('-') && CH(off+1) == _T('>')) + if(off < *p_scan_horizon && *p_scan_horizon >= max_end - len) { + /* We have already scanned the range up to the max_end so we know + * there is nothing to see. */ return FALSE; + } - while(1) { - while(off + 2 < lines[i].end) { - if(CH(off) == _T('-') && CH(off+1) == _T('-')) { - if(CH(off+2) == _T('>')) { - /* Success. */ - off += 2; - goto done; - } else { - /* "--" is prohibited inside the comment. */ - return FALSE; - } + while(TRUE) { + while(off + len <= lines[i].end && off + len <= max_end) { + if(md_ascii_eq(STR(off), str, len)) { + /* Success. */ + *p_end = off + len; + return TRUE; } - off++; } i++; - if(i >= n_lines) + if(off >= max_end || i >= n_lines) { + /* Failure. */ + *p_scan_horizon = off; return FALSE; + } off = lines[i].beg; - - if(off >= max_end) - return FALSE; } - -done: - if(off >= max_end) - return FALSE; - - *p_end = off+1; - return TRUE; } static int -md_is_html_processing_instruction(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) +md_is_html_comment(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) { OFF off = beg; - int i = 0; MD_ASSERT(CH(beg) == _T('<')); - if(off + 2 >= lines[0].end) + if(off + 4 >= lines[0].end) return FALSE; - if(CH(off+1) != _T('?')) + if(CH(off+1) != _T('!') || CH(off+2) != _T('-') || CH(off+3) != _T('-')) return FALSE; - off += 2; + off += 4; - while(1) { - while(off + 1 < lines[i].end) { - if(CH(off) == _T('?') && CH(off+1) == _T('>')) { - /* Success. */ - off++; - goto done; - } + /* ">" and "->" must not follow the opening. */ + if(off < lines[0].end && CH(off) == _T('>')) + return FALSE; + if(off+1 < lines[0].end && CH(off) == _T('-') && CH(off+1) == _T('>')) + return FALSE; - off++; + /* HTML comment must not contyain "--", so we scan just for "--" instead + * of "-->" and verify manually that '>' follows. */ + if(md_scan_for_html_closer(ctx, _T("--"), 2, + lines, n_lines, off, max_end, p_end, &ctx->html_comment_horizon)) + { + if(*p_end < max_end && CH(*p_end) == _T('>')) { + *p_end = *p_end + 1; + return TRUE; } + } - i++; - if(i >= n_lines) - return FALSE; + return FALSE; +} - off = lines[i].beg; - if(off >= max_end) - return FALSE; - } +static int +md_is_html_processing_instruction(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) +{ + OFF off = beg; -done: - if(off >= max_end) + if(off + 2 >= lines[0].end) + return FALSE; + if(CH(off+1) != _T('?')) return FALSE; + off += 2; - *p_end = off+1; - return TRUE; + return md_scan_for_html_closer(ctx, _T("?>"), 2, + lines, n_lines, off, max_end, p_end, &ctx->html_proc_instr_horizon); } static int md_is_html_declaration(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) { OFF off = beg; - int i = 0; - - MD_ASSERT(CH(beg) == _T('<')); if(off + 2 >= lines[0].end) return FALSE; @@ -1182,31 +1177,8 @@ md_is_html_declaration(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, if(off < lines[0].end && !ISWHITESPACE(off)) return FALSE; - while(1) { - while(off < lines[i].end) { - if(CH(off) == _T('>')) { - /* Success. */ - goto done; - } - - off++; - } - - i++; - if(i >= n_lines) - return FALSE; - - off = lines[i].beg; - if(off >= max_end) - return FALSE; - } - -done: - if(off >= max_end) - return FALSE; - - *p_end = off+1; - return TRUE; + return md_scan_for_html_closer(ctx, _T(">"), 1, + lines, n_lines, off, max_end, p_end, &ctx->html_decl_horizon); } static int @@ -1216,7 +1188,6 @@ md_is_html_cdata(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF ma static const SZ open_size = SIZEOF_ARRAY(open_str) - 1; OFF off = beg; - int i = 0; if(off + open_size >= lines[0].end) return FALSE; @@ -1224,49 +1195,22 @@ md_is_html_cdata(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF ma return FALSE; off += open_size; - while(1) { - while(off + 2 < lines[i].end) { - if(CH(off) == _T(']') && CH(off+1) == _T(']') && CH(off+2) == _T('>')) { - /* Success. */ - off += 2; - goto done; - } - - off++; - } - - i++; - if(i >= n_lines) - return FALSE; - - off = lines[i].beg; - if(off >= max_end) - return FALSE; - } - -done: - if(off >= max_end) - return FALSE; + if(lines[n_lines-1].end < max_end) + max_end = lines[n_lines-1].end - 2; - *p_end = off+1; - return TRUE; + return md_scan_for_html_closer(ctx, _T("]]>"), 3, + lines, n_lines, off, max_end, p_end, &ctx->html_cdata_horizon); } static int md_is_html_any(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF max_end, OFF* p_end) { - if(md_is_html_tag(ctx, lines, n_lines, beg, max_end, p_end) == TRUE) - return TRUE; - if(md_is_html_comment(ctx, lines, n_lines, beg, max_end, p_end) == TRUE) - return TRUE; - if(md_is_html_processing_instruction(ctx, lines, n_lines, beg, max_end, p_end) == TRUE) - return TRUE; - if(md_is_html_declaration(ctx, lines, n_lines, beg, max_end, p_end) == TRUE) - return TRUE; - if(md_is_html_cdata(ctx, lines, n_lines, beg, max_end, p_end) == TRUE) - return TRUE; - - return FALSE; + MD_ASSERT(CH(beg) == _T('<')); + return (md_is_html_tag(ctx, lines, n_lines, beg, max_end, p_end) || + md_is_html_comment(ctx, lines, n_lines, beg, max_end, p_end) || + md_is_html_processing_instruction(ctx, lines, n_lines, beg, max_end, p_end) || + md_is_html_declaration(ctx, lines, n_lines, beg, max_end, p_end) || + md_is_html_cdata(ctx, lines, n_lines, beg, max_end, p_end)); } @@ -1282,7 +1226,7 @@ md_is_hex_entity_contents(MD_CTX* ctx, const CHAR* text, OFF beg, OFF max_end, O while(off < max_end && ISXDIGIT_(text[off]) && off - beg <= 8) off++; - if(1 <= off - beg && off - beg <= 8) { + if(1 <= off - beg && off - beg <= 6) { *p_end = off; return TRUE; } else { @@ -1298,7 +1242,7 @@ md_is_dec_entity_contents(MD_CTX* ctx, const CHAR* text, OFF beg, OFF max_end, O while(off < max_end && ISDIGIT_(text[off]) && off - beg <= 8) off++; - if(1 <= off - beg && off - beg <= 8) { + if(1 <= off - beg && off - beg <= 7) { *p_end = off; return TRUE; } else { @@ -1551,7 +1495,7 @@ md_link_label_hash(const CHAR* label, SZ size) { unsigned hash = MD_FNV1A_BASE; OFF off; - int codepoint; + unsigned codepoint; int is_whitespace = FALSE; off = md_skip_unicode_whitespace(label, 0, size); @@ -1563,25 +1507,53 @@ md_link_label_hash(const CHAR* label, SZ size) if(is_whitespace) { codepoint = ' '; - hash = md_fnv1a(hash, &codepoint, 1 * sizeof(int)); - + hash = md_fnv1a(hash, &codepoint, sizeof(unsigned)); off = md_skip_unicode_whitespace(label, off, size); } else { MD_UNICODE_FOLD_INFO fold_info; md_get_unicode_fold_info(codepoint, &fold_info); - hash = md_fnv1a(hash, fold_info.codepoints, fold_info.n_codepoints * sizeof(int)); - + hash = md_fnv1a(hash, fold_info.codepoints, fold_info.n_codepoints * sizeof(unsigned)); off += char_size; } } - if(!is_whitespace) { - codepoint = ' '; - hash = md_fnv1a(hash, &codepoint, 1 * sizeof(int)); + return hash; +} + +static OFF +md_link_label_cmp_load_fold_info(const CHAR* label, OFF off, SZ size, + MD_UNICODE_FOLD_INFO* fold_info) +{ + unsigned codepoint; + SZ char_size; + + if(off >= size) { + /* Treat end of link label as a whitespace. */ + goto whitespace; + } + + if(ISNEWLINE_(label[off])) { + /* Treat new lines as a whitespace. */ + off++; + goto whitespace; } - return hash; + codepoint = md_decode_unicode(label, off, size, &char_size); + off += char_size; + if(ISUNICODEWHITESPACE_(codepoint)) { + /* Treat all whitespace as equivalent */ + goto whitespace; + } + + /* Get real folding info. */ + md_get_unicode_fold_info(codepoint, fold_info); + return off; + +whitespace: + fold_info->codepoints[0] = _T(' '); + fold_info->n_codepoints = 1; + return off; } static int @@ -1589,55 +1561,35 @@ md_link_label_cmp(const CHAR* a_label, SZ a_size, const CHAR* b_label, SZ b_size { OFF a_off; OFF b_off; + int a_reached_end = FALSE; + int b_reached_end = FALSE; + MD_UNICODE_FOLD_INFO a_fi = { 0 }; + MD_UNICODE_FOLD_INFO b_fi = { 0 }; + OFF a_fi_off = 0; + OFF b_fi_off = 0; + int cmp; - /* The slow path, with Unicode case folding and Unicode whitespace collapsing. */ a_off = md_skip_unicode_whitespace(a_label, 0, a_size); b_off = md_skip_unicode_whitespace(b_label, 0, b_size); - while(a_off < a_size || b_off < b_size) { - int a_codepoint, b_codepoint; - SZ a_char_size, b_char_size; - int a_is_whitespace, b_is_whitespace; - - if(a_off < a_size) { - a_codepoint = md_decode_unicode(a_label, a_off, a_size, &a_char_size); - a_is_whitespace = ISUNICODEWHITESPACE_(a_codepoint) || ISNEWLINE_(a_label[a_off]); - } else { - /* Treat end of label as a whitespace. */ - a_codepoint = -1; - a_is_whitespace = TRUE; + while(!a_reached_end && !b_reached_end) { + /* If needed, load fold info for next char. */ + if(a_fi_off >= a_fi.n_codepoints) { + a_fi_off = 0; + a_off = md_link_label_cmp_load_fold_info(a_label, a_off, a_size, &a_fi); + a_reached_end = (a_off >= a_size); } - - if(b_off < b_size) { - b_codepoint = md_decode_unicode(b_label, b_off, b_size, &b_char_size); - b_is_whitespace = ISUNICODEWHITESPACE_(b_codepoint) || ISNEWLINE_(b_label[b_off]); - } else { - /* Treat end of label as a whitespace. */ - b_codepoint = -1; - b_is_whitespace = TRUE; + if(b_fi_off >= b_fi.n_codepoints) { + b_fi_off = 0; + b_off = md_link_label_cmp_load_fold_info(b_label, b_off, b_size, &b_fi); + b_reached_end = (b_off >= b_size); } - if(a_is_whitespace || b_is_whitespace) { - if(!a_is_whitespace || !b_is_whitespace) - return (a_is_whitespace ? -1 : +1); - - a_off = md_skip_unicode_whitespace(a_label, a_off, a_size); - b_off = md_skip_unicode_whitespace(b_label, b_off, b_size); - } else { - MD_UNICODE_FOLD_INFO a_fold_info, b_fold_info; - int cmp; - - md_get_unicode_fold_info(a_codepoint, &a_fold_info); - md_get_unicode_fold_info(b_codepoint, &b_fold_info); + cmp = b_fi.codepoints[b_fi_off] - a_fi.codepoints[a_fi_off]; + if(cmp != 0) + return cmp; - if(a_fold_info.n_codepoints != b_fold_info.n_codepoints) - return (a_fold_info.n_codepoints - b_fold_info.n_codepoints); - cmp = memcmp(a_fold_info.codepoints, b_fold_info.codepoints, a_fold_info.n_codepoints * sizeof(int)); - if(cmp != 0) - return cmp; - - a_off += a_char_size; - b_off += b_char_size; - } + a_fi_off++; + b_fi_off++; } return 0; @@ -1911,7 +1863,7 @@ md_is_link_label(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, return FALSE; } } else { - int codepoint; + unsigned codepoint; SZ char_size; codepoint = md_decode_unicode(ctx->text, off, ctx->size, &char_size); @@ -1958,7 +1910,7 @@ md_is_link_destination_A(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end, continue; } - if(ISWHITESPACE(off) || CH(off) == _T('<')) + if(ISNEWLINE(off) || CH(off) == _T('<')) return FALSE; if(CH(off) == _T('>')) { @@ -2017,6 +1969,16 @@ md_is_link_destination_B(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end, return TRUE; } +static inline int +md_is_link_destination(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end, + OFF* p_contents_beg, OFF* p_contents_end) +{ + if(CH(beg) == _T('<')) + return md_is_link_destination_A(ctx, beg, max_end, p_end, p_contents_beg, p_contents_end); + else + return md_is_link_destination_B(ctx, beg, max_end, p_end, p_contents_beg, p_contents_end); +} + static int md_is_link_title(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF* p_end, int* p_beg_line_index, int* p_end_line_index, @@ -2026,7 +1988,7 @@ md_is_link_title(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, CHAR closer_char; int line_index = 0; - /* Optional white space with up to one line break. */ + /* White space with up to one line break. */ while(off < lines[line_index].end && ISWHITESPACE(off)) off++; if(off >= lines[line_index].end) { @@ -2035,6 +1997,8 @@ md_is_link_title(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, return FALSE; off = lines[line_index].beg; } + if(off == beg) + return FALSE; *p_beg_line_index = line_index; @@ -2061,6 +2025,9 @@ md_is_link_title(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, *p_end = off+1; *p_end_line_index = line_index; return TRUE; + } else if(closer_char == _T(')') && CH(off) == _T('(')) { + /* ()-style title cannot contain (unescaped '(')) */ + return FALSE; } off++; @@ -2081,9 +2048,7 @@ md_is_link_title(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, * Returns -1 in case of an error (out of memory). */ static int -md_is_link_reference_definition_helper( - MD_CTX* ctx, const MD_LINE* lines, int n_lines, - int (*is_link_dest_fn)(MD_CTX*, OFF, OFF, OFF*, OFF*, OFF*)) +md_is_link_reference_definition(MD_CTX* ctx, const MD_LINE* lines, int n_lines) { OFF label_contents_beg; OFF label_contents_end; @@ -2127,8 +2092,8 @@ md_is_link_reference_definition_helper( } /* Link destination. */ - if(!is_link_dest_fn(ctx, off, lines[line_index].end, - &off, &dest_contents_beg, &dest_contents_end)) + if(!md_is_link_destination(ctx, off, lines[line_index].end, + &off, &dest_contents_beg, &dest_contents_end)) return FALSE; /* (Optional) title. Note we interpret it as an title only if nothing @@ -2214,16 +2179,6 @@ abort: return -1; } -static inline int -md_is_link_reference_definition(MD_CTX* ctx, const MD_LINE* lines, int n_lines) -{ - int ret; - ret = md_is_link_reference_definition_helper(ctx, lines, n_lines, md_is_link_destination_A); - if(ret == 0) - ret = md_is_link_reference_definition_helper(ctx, lines, n_lines, md_is_link_destination_B); - return ret; -} - static int md_is_link_reference(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, OFF end, MD_LINK_ATTR* attr) @@ -2279,9 +2234,8 @@ abort: } static int -md_is_inline_link_spec_helper(MD_CTX* ctx, const MD_LINE* lines, int n_lines, - OFF beg, OFF* p_end, MD_LINK_ATTR* attr, - int (*is_link_dest_fn)(MD_CTX*, OFF, OFF, OFF*, OFF*, OFF*)) +md_is_inline_link_spec(MD_CTX* ctx, const MD_LINE* lines, int n_lines, + OFF beg, OFF* p_end, MD_LINK_ATTR* attr) { int line_index = 0; int tmp_line_index; @@ -2321,7 +2275,7 @@ md_is_inline_link_spec_helper(MD_CTX* ctx, const MD_LINE* lines, int n_lines, } /* Link destination. */ - if(!is_link_dest_fn(ctx, off, lines[line_index].end, + if(!md_is_link_destination(ctx, off, lines[line_index].end, &off, &attr->dest_beg, &attr->dest_end)) return FALSE; @@ -2376,14 +2330,6 @@ abort: return ret; } -static inline int -md_is_inline_link_spec(MD_CTX* ctx, const MD_LINE* lines, int n_lines, - OFF beg, OFF* p_end, MD_LINK_ATTR* attr) -{ - return md_is_inline_link_spec_helper(ctx, lines, n_lines, beg, p_end, attr, md_is_link_destination_A) || - md_is_inline_link_spec_helper(ctx, lines, n_lines, beg, p_end, attr, md_is_link_destination_B); -} - static void md_free_ref_defs(MD_CTX* ctx) { @@ -2493,13 +2439,42 @@ struct MD_MARK_tag { /* Mark flags specific for various mark types (so they can share bits). */ #define MD_MARK_EMPH_INTRAWORD 0x20 /* Helper for the "rule of 3". */ -#define MD_MARK_EMPH_MODULO3_0 0x40 -#define MD_MARK_EMPH_MODULO3_1 0x80 -#define MD_MARK_EMPH_MODULO3_2 (0x40 | 0x80) -#define MD_MARK_EMPH_MODULO3_MASK (0x40 | 0x80) +#define MD_MARK_EMPH_MOD3_0 0x40 +#define MD_MARK_EMPH_MOD3_1 0x80 +#define MD_MARK_EMPH_MOD3_2 (0x40 | 0x80) +#define MD_MARK_EMPH_MOD3_MASK (0x40 | 0x80) #define MD_MARK_AUTOLINK 0x20 /* Distinguisher for '<', '>'. */ #define MD_MARK_VALIDPERMISSIVEAUTOLINK 0x20 /* For permissive autolinks. */ +static MD_MARKCHAIN* +md_asterisk_chain(MD_CTX* ctx, unsigned flags) +{ + switch(flags & (MD_MARK_EMPH_INTRAWORD | MD_MARK_EMPH_MOD3_MASK)) { + case MD_MARK_EMPH_INTRAWORD | MD_MARK_EMPH_MOD3_0: return &ASTERISK_OPENERS_intraword_mod3_0; + case MD_MARK_EMPH_INTRAWORD | MD_MARK_EMPH_MOD3_1: return &ASTERISK_OPENERS_intraword_mod3_1; + case MD_MARK_EMPH_INTRAWORD | MD_MARK_EMPH_MOD3_2: return &ASTERISK_OPENERS_intraword_mod3_2; + case MD_MARK_EMPH_MOD3_0: return &ASTERISK_OPENERS_extraword_mod3_0; + case MD_MARK_EMPH_MOD3_1: return &ASTERISK_OPENERS_extraword_mod3_1; + case MD_MARK_EMPH_MOD3_2: return &ASTERISK_OPENERS_extraword_mod3_2; + default: MD_UNREACHABLE(); + } + return NULL; +} + +static MD_MARKCHAIN* +md_mark_chain(MD_CTX* ctx, int mark_index) +{ + MD_MARK* mark = &ctx->marks[mark_index]; + + switch(mark->ch) { + case _T('*'): return md_asterisk_chain(ctx, mark->flags); + case _T('_'): return &UNDERSCORE_OPENERS; + case _T('~'): return &TILDE_OPENERS; + case _T('['): return &BRACKET_OPENERS; + case _T('|'): return &TABLECELLBOUNDARIES; + default: return NULL; + } +} static MD_MARK* md_push_mark(MD_CTX* ctx) @@ -2658,18 +2633,11 @@ md_rollback(MD_CTX* ctx, int opener_index, int closer_index, int how) MD_MARKCHAIN* chain; mark_opener->flags &= ~(MD_MARK_OPENER | MD_MARK_CLOSER | MD_MARK_RESOLVED); - - switch(mark_opener->ch) { - case '*': chain = &ASTERISK_OPENERS; break; - case '_': chain = &UNDERSCORE_OPENERS; break; - case '`': chain = &BACKTICK_OPENERS; break; - case '<': chain = &LOWERTHEN_OPENERS; break; - case '~': chain = &TILDE_OPENERS; break; - default: MD_UNREACHABLE(); break; + chain = md_mark_chain(ctx, opener_index); + if(chain != NULL) { + md_mark_chain_append(ctx, chain, mark_opener_index); + discard_flag = 1; } - md_mark_chain_append(ctx, chain, mark_opener_index); - - discard_flag = 1; } } @@ -2730,19 +2698,250 @@ md_build_mark_char_map(MD_CTX* ctx) if(ctx->parser.flags & MD_FLAG_COLLAPSEWHITESPACE) { int i; - for(i = 0; i < sizeof(ctx->mark_char_map); i++) { + for(i = 0; i < (int) sizeof(ctx->mark_char_map); i++) { if(ISWHITESPACE_(i)) ctx->mark_char_map[i] = 1; } } } +/* We limit code span marks to lower then 32 backticks. This solves the + * pathologic case of too many openers, each of different length: Their + * resolving would be then O(n^2). */ +#define CODESPAN_MARK_MAXLEN 32 + +static int +md_is_code_span(MD_CTX* ctx, const MD_LINE* lines, int n_lines, OFF beg, + OFF* p_opener_beg, OFF* p_opener_end, + OFF* p_closer_beg, OFF* p_closer_end, + OFF last_potential_closers[CODESPAN_MARK_MAXLEN], + int* p_reached_paragraph_end) +{ + OFF opener_beg = beg; + OFF opener_end; + OFF closer_beg; + OFF closer_end; + SZ mark_len; + OFF line_end; + int has_space_after_opener = FALSE; + int has_eol_after_opener = FALSE; + int has_space_before_closer = FALSE; + int has_eol_before_closer = FALSE; + int has_only_space = TRUE; + int line_index = 0; + + line_end = lines[0].end; + opener_end = opener_beg; + while(opener_end < line_end && CH(opener_end) == _T('`')) + opener_end++; + has_space_after_opener = (opener_end < line_end && CH(opener_end) == _T(' ')); + has_eol_after_opener = (opener_end == line_end); + + /* The caller needs to know end of the opening mark even if we fail. */ + *p_opener_end = opener_end; + + mark_len = opener_end - opener_beg; + if(mark_len > CODESPAN_MARK_MAXLEN) + return FALSE; + + /* Check whether we already know there is no closer of this length. + * If so, re-scan does no sense. This fixes issue #59. */ + if(last_potential_closers[mark_len-1] >= lines[n_lines-1].end || + (*p_reached_paragraph_end && last_potential_closers[mark_len-1] < opener_end)) + return FALSE; + + closer_beg = opener_end; + closer_end = opener_end; + + /* Find closer mark. */ + while(TRUE) { + while(closer_beg < line_end && CH(closer_beg) != _T('`')) { + if(CH(closer_beg) != _T(' ')) + has_only_space = FALSE; + closer_beg++; + } + closer_end = closer_beg; + while(closer_end < line_end && CH(closer_end) == _T('`')) + closer_end++; + + if(closer_end - closer_beg == mark_len) { + /* Success. */ + has_space_before_closer = (closer_beg > lines[line_index].beg && CH(closer_beg-1) == _T(' ')); + has_eol_before_closer = (closer_beg == lines[line_index].beg); + break; + } + + if(closer_end - closer_beg > 0) { + /* We have found a back-tick which is not part of the closer. */ + has_only_space = FALSE; + + /* But if we eventually fail, remember it as a potential closer + * of its own length for future attempts. This mitigates needs for + * rescans. */ + if(closer_end - closer_beg < CODESPAN_MARK_MAXLEN) { + if(closer_beg > last_potential_closers[closer_end - closer_beg - 1]) + last_potential_closers[closer_end - closer_beg - 1] = closer_beg; + } + } + + if(closer_end >= line_end) { + line_index++; + if(line_index >= n_lines) { + /* Reached end of the paragraph and still nothing. */ + *p_reached_paragraph_end = TRUE; + return FALSE; + } + /* Try on the next line. */ + line_end = lines[line_index].end; + closer_beg = lines[line_index].beg; + } else { + closer_beg = closer_end; + } + } + + /* If there is a space or a new line both after and before the opener + * (and if the code span is not made of spaces only), consume one initial + * and one trailing space as part of the marks. */ + if(!has_only_space && + (has_space_after_opener || has_eol_after_opener) && + (has_space_before_closer || has_eol_before_closer)) + { + if(has_space_after_opener) + opener_end++; + else + opener_end = lines[1].beg; + + if(has_space_before_closer) + closer_beg--; + else { + closer_beg = lines[line_index-1].end; + /* We need to eat the preceding "\r\n" but not any line trailing + * spaces. */ + while(closer_beg < ctx->size && ISBLANK(closer_beg)) + closer_beg++; + } + } + + *p_opener_beg = opener_beg; + *p_opener_end = opener_end; + *p_closer_beg = closer_beg; + *p_closer_end = closer_end; + return TRUE; +} + +static int +md_is_autolink_uri(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end) +{ + OFF off = beg+1; + + MD_ASSERT(CH(beg) == _T('<')); + + /* Check for scheme. */ + if(off >= max_end || !ISASCII(off)) + return FALSE; + off++; + while(1) { + if(off >= max_end) + return FALSE; + if(off - beg > 32) + return FALSE; + if(CH(off) == _T(':') && off - beg >= 3) + break; + if(!ISALNUM(off) && CH(off) != _T('+') && CH(off) != _T('-') && CH(off) != _T('.')) + return FALSE; + off++; + } + + /* Check the path after the scheme. */ + while(off < max_end && CH(off) != _T('>')) { + if(ISWHITESPACE(off) || ISCNTRL(off) || CH(off) == _T('<')) + return FALSE; + off++; + } + + if(off >= max_end) + return FALSE; + + MD_ASSERT(CH(off) == _T('>')); + *p_end = off+1; + return TRUE; +} + +static int +md_is_autolink_email(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end) +{ + OFF off = beg + 1; + int label_len; + + MD_ASSERT(CH(beg) == _T('<')); + + /* The code should correspond to this regexp: + /^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+ + @[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])? + (?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ + */ + + /* Username (before '@'). */ + while(off < max_end && (ISALNUM(off) || ISANYOF(off, _T(".!#$%&'*+/=?^_`{|}~-")))) + off++; + if(off <= beg+1) + return FALSE; + + /* '@' */ + if(off >= max_end || CH(off) != _T('@')) + return FALSE; + off++; + + /* Labels delimited with '.'; each label is sequence of 1 - 62 alnum + * characters or '-', but '-' is not allowed as first or last char. */ + label_len = 0; + while(off < max_end) { + if(ISALNUM(off)) + label_len++; + else if(CH(off) == _T('-') && label_len > 0) + label_len++; + else if(CH(off) == _T('.') && label_len > 0 && CH(off-1) != _T('-')) + label_len = 0; + else + break; + + if(label_len > 62) + return FALSE; + + off++; + } + + if(label_len <= 0 || off >= max_end || CH(off) != _T('>') || CH(off-1) == _T('-')) + return FALSE; + + *p_end = off+1; + return TRUE; +} + +static int +md_is_autolink(MD_CTX* ctx, OFF beg, OFF max_end, OFF* p_end, int* p_missing_mailto) +{ + if(md_is_autolink_uri(ctx, beg, max_end, p_end)) { + *p_missing_mailto = FALSE; + return TRUE; + } + + if(md_is_autolink_email(ctx, beg, max_end, p_end)) { + *p_missing_mailto = TRUE; + return TRUE; + } + + return FALSE; +} + static int md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) { int i; int ret = 0; MD_MARK* mark; + OFF codespan_last_potential_closers[CODESPAN_MARK_MAXLEN] = { 0 }; + int codespan_scanned_till_paragraph_end = FALSE; for(i = 0; i < n_lines; i++) { const MD_LINE* line = &lines[i]; @@ -2761,8 +2960,8 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) #define IS_MARK_CHAR(off) (ctx->mark_char_map[(unsigned char) CH(off)]) #endif - /* Optimization: Fast path (with some loop unrolling). */ - while(off + 4 < line_end && !IS_MARK_CHAR(off+0) && !IS_MARK_CHAR(off+1) + /* Optimization: Use some loop unrolling. */ + while(off + 3 < line_end && !IS_MARK_CHAR(off+0) && !IS_MARK_CHAR(off+1) && !IS_MARK_CHAR(off+2) && !IS_MARK_CHAR(off+3)) off += 4; while(off < line_end && !IS_MARK_CHAR(off+0)) @@ -2780,14 +2979,7 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) /* Hard-break cannot be on the last line of the block. */ if(!ISNEWLINE(off+1) || i+1 < n_lines) PUSH_MARK(ch, off, off+2, MD_MARK_RESOLVED); - - /* If '`' or '>' follows, we need both marks as the backslash - * may be inside a code span or an autolink where escaping is - * disabled. */ - if(CH(off+1) == _T('`') || CH(off+1) == _T('>')) - off++; - else - off += 2; + off += 2; continue; } @@ -2835,9 +3027,9 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) * split the mark when being later resolved partially by some * shorter closer. */ switch((tmp - off) % 3) { - case 0: flags |= MD_MARK_EMPH_MODULO3_0; break; - case 1: flags |= MD_MARK_EMPH_MODULO3_1; break; - case 2: flags |= MD_MARK_EMPH_MODULO3_2; break; + case 0: flags |= MD_MARK_EMPH_MOD3_0; break; + case 1: flags |= MD_MARK_EMPH_MOD3_1; break; + case 2: flags |= MD_MARK_EMPH_MOD3_2; break; } PUSH_MARK(ch, off, tmp, flags); @@ -2860,18 +3052,32 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) /* A potential code span start/end. */ if(ch == _T('`')) { - OFF tmp = off+1; - - while(tmp < line_end && CH(tmp) == _T('`')) - tmp++; - - /* We limit code span marks to lower then 256 backticks. This - * solves a pathologic case of too many openers, each of - * different length: Their resolving is then O(n^2). */ - if(tmp - off < 256) - PUSH_MARK(ch, off, tmp, MD_MARK_POTENTIAL_OPENER | MD_MARK_POTENTIAL_CLOSER); + OFF opener_beg, opener_end; + OFF closer_beg, closer_end; + int is_code_span; + + is_code_span = md_is_code_span(ctx, lines + i, n_lines - i, off, + &opener_beg, &opener_end, &closer_beg, &closer_end, + codespan_last_potential_closers, + &codespan_scanned_till_paragraph_end); + if(is_code_span) { + PUSH_MARK(_T('`'), opener_beg, opener_end, MD_MARK_OPENER | MD_MARK_RESOLVED); + PUSH_MARK(_T('`'), closer_beg, closer_end, MD_MARK_CLOSER | MD_MARK_RESOLVED); + ctx->marks[ctx->n_marks-2].next = ctx->n_marks-1; + ctx->marks[ctx->n_marks-1].prev = ctx->n_marks-2; + + off = closer_end; + + /* Advance the current line accordingly. */ + while(off > line_end) { + i++; + line++; + line_end = line->end; + } + continue; + } - off = tmp; + off = opener_end; continue; } @@ -2893,9 +3099,49 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) } /* A potential autolink or raw HTML start/end. */ - if(ch == _T('<') || ch == _T('>')) { - if(!(ctx->parser.flags & MD_FLAG_NOHTMLSPANS)) - PUSH_MARK(ch, off, off+1, (ch == _T('<') ? MD_MARK_POTENTIAL_OPENER : MD_MARK_POTENTIAL_CLOSER)); + if(ch == _T('<')) { + int is_autolink; + OFF autolink_end; + int missing_mailto; + + if(!(ctx->parser.flags & MD_FLAG_NOHTMLSPANS)) { + int is_html; + OFF html_end; + + /* Given the nature of the raw HTML, we have to recognize + * it here. Doing so later in md_analyze_lt_gt() could + * open can of worms of quadratic complexity. */ + is_html = md_is_html_any(ctx, lines + i, n_lines - i, off, + lines[n_lines-1].end, &html_end); + if(is_html) { + PUSH_MARK(_T('<'), off, off, MD_MARK_OPENER | MD_MARK_RESOLVED); + PUSH_MARK(_T('>'), html_end, html_end, MD_MARK_CLOSER | MD_MARK_RESOLVED); + ctx->marks[ctx->n_marks-2].next = ctx->n_marks-1; + ctx->marks[ctx->n_marks-1].prev = ctx->n_marks-2; + off = html_end; + + /* Advance the current line accordingly. */ + while(off > line_end) { + i++; + line++; + line_end = line->end; + } + continue; + } + } + + is_autolink = md_is_autolink(ctx, off, lines[n_lines-1].end, + &autolink_end, &missing_mailto); + if(is_autolink) { + PUSH_MARK((missing_mailto ? _T('@') : _T('<')), off, off+1, + MD_MARK_OPENER | MD_MARK_RESOLVED | MD_MARK_AUTOLINK); + PUSH_MARK(_T('>'), autolink_end-1, autolink_end, + MD_MARK_CLOSER | MD_MARK_RESOLVED | MD_MARK_AUTOLINK); + ctx->marks[ctx->n_marks-2].next = ctx->n_marks-1; + ctx->marks[ctx->n_marks-1].prev = ctx->n_marks-2; + off = autolink_end; + continue; + } off++; continue; @@ -2947,7 +3193,7 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) }; int scheme_index; - for(scheme_index = 0; scheme_index < SIZEOF_ARRAY(scheme_map); scheme_index++) { + for(scheme_index = 0; scheme_index < (int) SIZEOF_ARRAY(scheme_map); scheme_index++) { const CHAR* scheme = scheme_map[scheme_index].scheme; const SZ scheme_size = scheme_map[scheme_index].scheme_size; const CHAR* suffix = scheme_map[scheme_index].suffix; @@ -3002,6 +3248,7 @@ md_collect_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mode) PUSH_MARK(ch, off, tmp, MD_MARK_POTENTIAL_OPENER | MD_MARK_POTENTIAL_CLOSER); off = tmp; + continue; } /* Turn non-trivial whitespace into single space. */ @@ -3037,231 +3284,6 @@ abort: return ret; } - -/* Analyze whether the back-tick is really start/end mark of a code span. - * If yes, reset all marks inside of it and setup flags of both marks. */ -static void -md_analyze_backtick(MD_CTX* ctx, int mark_index) -{ - MD_MARK* mark = &ctx->marks[mark_index]; - int opener_index = BACKTICK_OPENERS.tail; - - /* Try to find unresolved opener of the same length. If we find it, - * we form a code span. */ - while(opener_index >= 0) { - MD_MARK* opener = &ctx->marks[opener_index]; - - if(opener->end - opener->beg == mark->end - mark->beg) { - /* Rollback anything found inside it. - * (e.g. the code span contains some back-ticks or other special - * chars we misinterpreted.) */ - md_rollback(ctx, opener_index, mark_index, MD_ROLLBACK_ALL); - - /* Resolve the span. */ - md_resolve_range(ctx, &BACKTICK_OPENERS, opener_index, mark_index); - - /* Append any space or new line inside the span into the mark - * itself to swallow it. */ - while(CH(opener->end) == _T(' ') || ISNEWLINE(opener->end)) - opener->end++; - if(mark->beg > opener->end) { - while(CH(mark->beg-1) == _T(' ') || ISNEWLINE(mark->beg-1)) - mark->beg--; - } - - /* Done. */ - return; - } - - opener_index = ctx->marks[opener_index].prev; - } - - /* We didn't find any matching opener, so we ourselves may be the opener - * of some upcoming closer. We also have to handle specially if there is - * a backslash mark before it as that can cancel the first backtick. */ - if(mark_index > 0 && (mark-1)->beg == mark->beg - 1 && (mark-1)->ch == '\\') { - if(mark->end - mark->beg == 1) { - /* Single escaped backtick. */ - return; - } - - /* Remove the escaped backtick from the opener. */ - mark->beg++; - } - - if(mark->flags & MD_MARK_POTENTIAL_OPENER) - md_mark_chain_append(ctx, &BACKTICK_OPENERS, mark_index); -} - -static int -md_is_autolink_uri(MD_CTX* ctx, OFF beg, OFF end) -{ - OFF off = beg; - - /* Check for scheme. */ - if(off >= end || !ISASCII(off)) - return FALSE; - off++; - while(1) { - if(off >= end) - return FALSE; - if(off - beg > 32) - return FALSE; - if(CH(off) == _T(':') && off - beg >= 2) - break; - if(!ISALNUM(off) && CH(off) != _T('+') && CH(off) != _T('-') && CH(off) != _T('.')) - return FALSE; - off++; - } - - /* Check the path after the scheme. */ - while(off < end) { - if(ISWHITESPACE(off) || ISCNTRL(off) || CH(off) == _T('<') || CH(off) == _T('>')) - return FALSE; - off++; - } - - return TRUE; -} - -static int -md_is_autolink_email(MD_CTX* ctx, OFF beg, OFF end) -{ - OFF off = beg; - int label_len; - - /* The code should correspond to this regexp: - /^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+ - @[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])? - (?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/ - */ - - /* Username (before '@'). */ - while(off < end && (ISALNUM(off) || ISANYOF(off, _T(".!#$%&'*+/=?^_`{|}~-")))) - off++; - if(off <= beg) - return FALSE; - - /* '@' */ - if(off >= end || CH(off) != _T('@')) - return FALSE; - off++; - - /* Labels delimited with '.'; each label is sequence of 1 - 62 alnum - * characters or '-', but '-' is not allowed as first or last char. */ - label_len = 0; - while(off < end) { - if(ISALNUM(off)) - label_len++; - else if(CH(off) == _T('-') && label_len > 0) - label_len++; - else if(CH(off) == _T('.') && label_len > 0 && CH(off-1) != _T('-')) - label_len = 0; - else - return FALSE; - - if(label_len > 63) - return FALSE; - - off++; - } - - if(label_len <= 0 || CH(off-1) == _T('-')) - return FALSE; - - return TRUE; -} - -static int -md_is_autolink(MD_CTX* ctx, OFF beg, OFF end, int* p_missing_mailto) -{ - MD_ASSERT(CH(beg) == _T('<')); - MD_ASSERT(CH(end-1) == _T('>')); - - beg++; - end--; - - if(md_is_autolink_uri(ctx, beg, end)) - return TRUE; - - if(md_is_autolink_email(ctx, beg, end)) { - *p_missing_mailto = 1; - return TRUE; - } - - return FALSE; -} - -static void -md_analyze_lt_gt(MD_CTX* ctx, int mark_index, const MD_LINE* lines, int n_lines) -{ - MD_MARK* mark = &ctx->marks[mark_index]; - int opener_index; - - /* If it is an opener ('<'), remember it. */ - if(mark->flags & MD_MARK_POTENTIAL_OPENER) { - md_mark_chain_append(ctx, &LOWERTHEN_OPENERS, mark_index); - return; - } - - /* Otherwise we are potential closer and we try to resolve with since all - * the chained unresolved openers. */ - opener_index = LOWERTHEN_OPENERS.head; - while(opener_index >= 0) { - MD_MARK* opener = &ctx->marks[opener_index]; - OFF detected_end; - int is_autolink = 0; - int is_missing_mailto = 0; - int is_raw_html = 0; - - is_autolink = (md_is_autolink(ctx, opener->beg, mark->end, &is_missing_mailto)); - - if(is_autolink) { - if(is_missing_mailto) - opener->ch = _T('@'); - } else { - /* Identify the line where the opening mark lives. */ - int line_index = 0; - while(1) { - if(opener->beg < lines[line_index].end) - break; - line_index++; - } - - is_raw_html = (md_is_html_any(ctx, lines + line_index, - n_lines - line_index, opener->beg, mark->end, &detected_end)); - } - - /* Check whether the range forms a valid raw HTML. */ - if(is_autolink || is_raw_html) { - md_rollback(ctx, opener_index, mark_index, MD_ROLLBACK_ALL); - md_resolve_range(ctx, &LOWERTHEN_OPENERS, opener_index, mark_index); - - if(is_raw_html) { - /* If this fails, it means we have missed some earlier opportunity - * to resolve the opener of raw HTML. */ - MD_ASSERT(detected_end == mark->end); - - /* Make these marks zero width so the '<' and '>' are part of its - * contents. */ - opener->end = opener->beg; - mark->beg = mark->end; - - opener->flags &= ~MD_MARK_AUTOLINK; - mark->flags &= ~MD_MARK_AUTOLINK; - } else { - opener->flags |= MD_MARK_AUTOLINK; - mark->flags |= MD_MARK_AUTOLINK; - } - - /* And we are done. */ - return; - } - - opener_index = opener->next; - } -} - static void md_analyze_bracket(MD_CTX* ctx, int mark_index) { @@ -3501,7 +3523,7 @@ md_analyze_table_cell_boundary(MD_CTX* ctx, int mark_index) * follows. */ static int -md_split_simple_pairing_mark(MD_CTX* ctx, int mark_index, SZ n) +md_split_emph_mark(MD_CTX* ctx, int mark_index, SZ n) { MD_MARK* mark = &ctx->marks[mark_index]; int new_mark_index = mark_index + (mark->end - mark->beg - n); @@ -3518,79 +3540,77 @@ md_split_simple_pairing_mark(MD_CTX* ctx, int mark_index, SZ n) } static void -md_analyze_simple_pairing_mark(MD_CTX* ctx, MD_MARKCHAIN* chain, int mark_index, - int apply_rule_of_three) +md_analyze_emph(MD_CTX* ctx, int mark_index) { MD_MARK* mark = &ctx->marks[mark_index]; + MD_MARKCHAIN* chain = md_mark_chain(ctx, mark_index); /* If we can be a closer, try to resolve with the preceding opener. */ - if((mark->flags & MD_MARK_POTENTIAL_CLOSER) && chain->tail >= 0) { - int opener_index = chain->tail; - MD_MARK* opener = &ctx->marks[opener_index]; - SZ opener_size = opener->end - opener->beg; - SZ closer_size = mark->end - mark->beg; - - /* Apply the "rule of three". */ - if(apply_rule_of_three) { - while((mark->flags & MD_MARK_EMPH_INTRAWORD) || (opener->flags & MD_MARK_EMPH_INTRAWORD)) { - SZ opener_orig_size_modulo3; - - switch(opener->flags & MD_MARK_EMPH_MODULO3_MASK) { - case MD_MARK_EMPH_MODULO3_0: opener_orig_size_modulo3 = 0; break; - case MD_MARK_EMPH_MODULO3_1: opener_orig_size_modulo3 = 1; break; - case MD_MARK_EMPH_MODULO3_2: opener_orig_size_modulo3 = 2; break; - default: MD_UNREACHABLE(); break; + if(mark->flags & MD_MARK_POTENTIAL_CLOSER) { + MD_MARK* opener = NULL; + int opener_index; + + if(mark->ch == _T('*')) { + MD_MARKCHAIN* opener_chains[6]; + int i, n_opener_chains; + unsigned flags = mark->flags; + + /* Apply "rule of three". (This is why we break asterisk opener + * marks into multiple chains.) */ + n_opener_chains = 0; + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_intraword_mod3_0; + if((flags & MD_MARK_EMPH_MOD3_MASK) != MD_MARK_EMPH_MOD3_2) + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_intraword_mod3_1; + if((flags & MD_MARK_EMPH_MOD3_MASK) != MD_MARK_EMPH_MOD3_1) + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_intraword_mod3_2; + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_extraword_mod3_0; + if(!(flags & MD_MARK_EMPH_INTRAWORD) || (flags & MD_MARK_EMPH_MOD3_MASK) != MD_MARK_EMPH_MOD3_2) + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_extraword_mod3_1; + if(!(flags & MD_MARK_EMPH_INTRAWORD) || (flags & MD_MARK_EMPH_MOD3_MASK) != MD_MARK_EMPH_MOD3_1) + opener_chains[n_opener_chains++] = &ASTERISK_OPENERS_extraword_mod3_2; + + /* Opener is the most recent mark from the allowed chains. */ + for(i = 0; i < n_opener_chains; i++) { + if(opener_chains[i]->tail >= 0) { + int tmp_index = opener_chains[i]->tail; + MD_MARK* tmp_mark = &ctx->marks[tmp_index]; + if(opener == NULL || tmp_mark->end > opener->end) { + opener_index = tmp_index; + opener = tmp_mark; + } } + } + } else { + /* Simple emph. mark */ + if(chain->tail >= 0) { + opener_index = chain->tail; + opener = &ctx->marks[opener_index]; + } + } - if((opener_orig_size_modulo3 + closer_size) % 3 != 0) { - /* This opener is suitable. */ - break; - } + /* Resolve, if we have found matching opener. */ + if(opener != NULL) { + SZ opener_size = opener->end - opener->beg; + SZ closer_size = mark->end - mark->beg; - if(opener->prev >= 0) { - /* Try previous opener. */ - opener_index = opener->prev; - opener = &ctx->marks[opener_index]; - opener_size = opener->end - opener->beg; - closer_size = mark->end - mark->beg; - } else { - /* No suitable opener found. */ - goto cannot_resolve; - } + if(opener_size > closer_size) { + opener_index = md_split_emph_mark(ctx, opener_index, closer_size); + md_mark_chain_append(ctx, md_mark_chain(ctx, opener_index), opener_index); + } else if(opener_size < closer_size) { + md_split_emph_mark(ctx, mark_index, closer_size - opener_size); } - } - if(opener_size > closer_size) { - opener_index = md_split_simple_pairing_mark(ctx, opener_index, closer_size); - md_mark_chain_append(ctx, chain, opener_index); - } else if(opener_size < closer_size) { - md_split_simple_pairing_mark(ctx, mark_index, closer_size - opener_size); + md_rollback(ctx, opener_index, mark_index, MD_ROLLBACK_CROSSING); + md_resolve_range(ctx, chain, opener_index, mark_index); + return; } - - md_rollback(ctx, opener_index, mark_index, MD_ROLLBACK_CROSSING); - md_resolve_range(ctx, chain, opener_index, mark_index); - return; } -cannot_resolve: - /* If not resolved, and we can be an opener, remember the mark for - * the future. */ + /* If we could not resolve as closer, we may be yet be an opener. */ if(mark->flags & MD_MARK_POTENTIAL_OPENER) md_mark_chain_append(ctx, chain, mark_index); } -static inline void -md_analyze_asterisk(MD_CTX* ctx, int mark_index) -{ - md_analyze_simple_pairing_mark(ctx, &ASTERISK_OPENERS, mark_index, 1); -} - -static inline void -md_analyze_underscore(MD_CTX* ctx, int mark_index) -{ - md_analyze_simple_pairing_mark(ctx, &UNDERSCORE_OPENERS, mark_index, 1); -} - static void md_analyze_tilde(MD_CTX* ctx, int mark_index) { @@ -3618,55 +3638,57 @@ md_analyze_permissive_url_autolink(MD_CTX* ctx, int mark_index) MD_MARK* closer = &ctx->marks[closer_index]; MD_MARK* next_resolved_mark; OFF off = opener->end; - int seen_dot = FALSE; - int seen_underscore_or_hyphen[2] = { FALSE, FALSE }; + int n_dots = FALSE; + int has_underscore_in_last_seg = FALSE; + int has_underscore_in_next_to_last_seg = FALSE; + int n_opened_parenthesis = 0; /* Check for domain. */ while(off < ctx->size) { - if(ISALNUM(off)) { + if(ISALNUM(off) || CH(off) == _T('-')) { off++; } else if(CH(off) == _T('.')) { - seen_dot = TRUE; - seen_underscore_or_hyphen[0] = seen_underscore_or_hyphen[1]; - seen_underscore_or_hyphen[1] = FALSE; + /* We must see at least one period. */ + n_dots++; + has_underscore_in_next_to_last_seg = has_underscore_in_last_seg; + has_underscore_in_last_seg = FALSE; off++; - } else if(ISANYOF2(off, _T('-'), _T('_'))) { - seen_underscore_or_hyphen[1] = TRUE; + } else if(CH(off) == _T('_')) { + /* No underscore may be present in the last two domain segments. */ + has_underscore_in_last_seg = TRUE; off++; } else { break; } } - - if(off <= opener->end || !seen_dot || seen_underscore_or_hyphen[0] || seen_underscore_or_hyphen[1]) + if(off > opener->end && CH(off-1) == _T('.')) { + off--; + n_dots--; + } + if(off <= opener->end || n_dots == 0 || has_underscore_in_next_to_last_seg || has_underscore_in_last_seg) return; /* Check for path. */ next_resolved_mark = closer + 1; while(next_resolved_mark->ch == 'D' || !(next_resolved_mark->flags & MD_MARK_RESOLVED)) next_resolved_mark++; - while(off < next_resolved_mark->beg && CH(off) != _T('<') && !ISWHITESPACE(off) && !ISNEWLINE(off)) - off++; - - /* Path validation. */ - if(ISANYOF(off-1, _T("?!.,:*_~)"))) { - if(CH(off-1) != _T(')')) { - off--; - } else { - int parenthesis_balance = 0; - OFF tmp; - - for(tmp = opener->end; tmp < off; tmp++) { - if(CH(tmp) == _T('(')) - parenthesis_balance++; - else if(CH(tmp) == _T(')')) - parenthesis_balance--; - } - - if(parenthesis_balance < 0) - off--; + while(off < next_resolved_mark->beg && CH(off) != _T('<') && !ISWHITESPACE(off) && !ISNEWLINE(off)) { + /* Parenthesis must be balanced. */ + if(CH(off) == _T('(')) { + n_opened_parenthesis++; + } else if(CH(off) == _T(')')) { + if(n_opened_parenthesis > 0) + n_opened_parenthesis--; + else + break; } + + off++; } + /* These cannot be last char In such case they are more likely normal + * punctuation. */ + if(ISANYOF(off-1, _T("?!.,:*_~"))) + off--; /* Ok. Lets call it auto-link. Adapt opener and create closer to zero * length so all the contents becomes the link text. */ @@ -3755,16 +3777,13 @@ md_analyze_marks(MD_CTX* ctx, const MD_LINE* lines, int n_lines, /* Analyze the mark. */ switch(mark->ch) { - case '`': md_analyze_backtick(ctx, i); break; - case '<': /* Pass through. */ - case '>': md_analyze_lt_gt(ctx, i, lines, n_lines); break; case '[': /* Pass through. */ case '!': /* Pass through. */ case ']': md_analyze_bracket(ctx, i); break; case '&': md_analyze_entity(ctx, i); break; case '|': md_analyze_table_cell_boundary(ctx, i); break; - case '*': md_analyze_asterisk(ctx, i); break; - case '_': md_analyze_underscore(ctx, i); break; + case '_': /* Pass through. */ + case '*': md_analyze_emph(ctx, i); break; case '~': md_analyze_tilde(ctx, i); break; case '.': /* Pass through. */ case ':': md_analyze_permissive_url_autolink(ctx, i); break; @@ -3789,11 +3808,7 @@ md_analyze_inlines(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int table_mod /* We analyze marks in few groups to handle their precedence. */ /* (1) Entities; code spans; autolinks; raw HTML. */ - md_analyze_marks(ctx, lines, n_lines, 0, ctx->n_marks, _T("&`<>")); - BACKTICK_OPENERS.head = -1; - BACKTICK_OPENERS.tail = -1; - LOWERTHEN_OPENERS.head = -1; - LOWERTHEN_OPENERS.tail = -1; + md_analyze_marks(ctx, lines, n_lines, 0, ctx->n_marks, _T("&")); if(table_mode) { /* (2) Analyze table cell boundaries. @@ -3827,8 +3842,18 @@ md_analyze_link_contents(MD_CTX* ctx, const MD_LINE* lines, int n_lines, int mark_beg, int mark_end) { md_analyze_marks(ctx, lines, n_lines, mark_beg, mark_end, _T("*_~@:.")); - ASTERISK_OPENERS.head = -1; - ASTERISK_OPENERS.tail = -1; + ASTERISK_OPENERS_extraword_mod3_0.head = -1; + ASTERISK_OPENERS_extraword_mod3_0.tail = -1; + ASTERISK_OPENERS_extraword_mod3_1.head = -1; + ASTERISK_OPENERS_extraword_mod3_1.tail = -1; + ASTERISK_OPENERS_extraword_mod3_2.head = -1; + ASTERISK_OPENERS_extraword_mod3_2.tail = -1; + ASTERISK_OPENERS_intraword_mod3_0.head = -1; + ASTERISK_OPENERS_intraword_mod3_0.tail = -1; + ASTERISK_OPENERS_intraword_mod3_1.head = -1; + ASTERISK_OPENERS_intraword_mod3_1.tail = -1; + ASTERISK_OPENERS_intraword_mod3_2.head = -1; + ASTERISK_OPENERS_intraword_mod3_2.tail = -1; UNDERSCORE_OPENERS.head = -1; UNDERSCORE_OPENERS.tail = -1; TILDE_OPENERS.head = -1; @@ -4048,14 +4073,24 @@ md_process_inlines(MD_CTX* ctx, const MD_LINE* lines, int n_lines) break; if(text_type == MD_TEXT_CODE) { - /* Inside code spans, new lines are transformed into single - * spaces. */ + OFF tmp; + MD_ASSERT(prev_mark != NULL); MD_ASSERT(prev_mark->ch == '`' && (prev_mark->flags & MD_MARK_OPENER)); MD_ASSERT(mark->ch == '`' && (mark->flags & MD_MARK_CLOSER)); + /* Inside a code span, trailing line whitespace has to be + * outputted. */ + tmp = off; + while(off < ctx->size && ISBLANK(off)) + off++; + if(off > tmp) + MD_TEXT(MD_TEXT_CODE, STR(tmp), off-tmp); + + /* and new lines are transformed into single spaces. */ if(prev_mark->end < off && off < mark->beg) MD_TEXT(MD_TEXT_CODE, _T(" "), 1); + } else if(text_type == MD_TEXT_HTML) { /* Inside raw HTML, we output the new line verbatim, including * any trailing spaces. */ @@ -4285,6 +4320,7 @@ abort: #define MD_BLOCK_CONTAINER_CLOSER 0x02 #define MD_BLOCK_CONTAINER (MD_BLOCK_CONTAINER_OPENER | MD_BLOCK_CONTAINER_CLOSER) #define MD_BLOCK_LOOSE_LIST 0x04 +#define MD_BLOCK_SETEXT_HEADER 0x08 struct MD_BLOCK_tag { MD_BLOCKTYPE type : 8; @@ -4351,7 +4387,7 @@ md_process_verbatim_block_contents(MD_CTX* ctx, MD_TEXTTYPE text_type, const MD_ MD_ASSERT(indent >= 0); /* Output code indentation. */ - while(indent > SIZEOF_ARRAY(indent_chunk_str)) { + while(indent > (int) SIZEOF_ARRAY(indent_chunk_str)) { MD_TEXT(text_type, indent_chunk_str, indent_chunk_size); indent -= SIZEOF_ARRAY(indent_chunk_str); } @@ -4425,6 +4461,8 @@ md_setup_fenced_code_detail(MD_CTX* ctx, const MD_BLOCK* block, MD_BLOCK_CODE_DE lang_end++; MD_CHECK(md_build_attribute(ctx, STR(beg), lang_end - beg, 0, &det->lang, lang_build)); + det->fence_char = fence_ch; + abort: return ret; } @@ -4715,6 +4753,7 @@ md_consume_link_reference_definitions(MD_CTX* ctx) /* Remove complete block. */ ctx->n_block_bytes -= n * sizeof(MD_LINE); ctx->n_block_bytes -= sizeof(MD_BLOCK); + ctx->current_block = NULL; } else { /* Remove just some initial lines from the block. */ memmove(lines, lines + n, (n_lines - n) * sizeof(MD_LINE)); @@ -4737,10 +4776,30 @@ md_end_current_block(MD_CTX* ctx) /* Check whether there is a reference definition. (We do this here instead * of in md_analyze_line() because reference definition can take multiple * lines.) */ - if(ctx->current_block->type == MD_BLOCK_P) { + if(ctx->current_block->type == MD_BLOCK_P || + (ctx->current_block->type == MD_BLOCK_H && (ctx->current_block->flags & MD_BLOCK_SETEXT_HEADER))) + { MD_LINE* lines = (MD_LINE*) (ctx->current_block + 1); - if(CH(lines[0].beg) == _T('[')) + if(CH(lines[0].beg) == _T('[')) { MD_CHECK(md_consume_link_reference_definitions(ctx)); + if(ctx->current_block == NULL) + return ret; + } + } + + if(ctx->current_block->type == MD_BLOCK_H && (ctx->current_block->flags & MD_BLOCK_SETEXT_HEADER)) { + int n_lines = ctx->current_block->n_lines; + + if(n_lines > 1) { + /* Get rid of the underline. */ + ctx->current_block->n_lines--; + ctx->n_block_bytes -= sizeof(MD_LINE); + } else { + /* Only the underline has left after eating the ref. defs. + * Keep the line as beginning of a new ordinary paragraph. */ + ctx->current_block->type = MD_BLOCK_P; + return 0; + } } /* Mark we are not building any block anymore. */ @@ -4809,7 +4868,7 @@ abort: ***********************/ static int -md_is_hr_line(MD_CTX* ctx, OFF beg, OFF* p_end) +md_is_hr_line(MD_CTX* ctx, OFF beg, OFF* p_end, OFF* p_killer) { OFF off = beg + 1; int n = 1; @@ -4820,12 +4879,16 @@ md_is_hr_line(MD_CTX* ctx, OFF beg, OFF* p_end) off++; } - if(n < 3) + if(n < 3) { + *p_killer = off; return FALSE; + } /* Nothing else can be present on the line. */ - if(off < ctx->size && !ISNEWLINE(off)) + if(off < ctx->size && !ISNEWLINE(off)) { + *p_killer = off; return FALSE; + } *p_end = off; return TRUE; @@ -4864,9 +4927,6 @@ md_is_setext_underline(MD_CTX* ctx, OFF beg, OFF* p_end, unsigned* p_level) while(off < ctx->size && CH(off) == CH(beg)) off++; - while(off < ctx->size && CH(off) == _T(' ')) - off++; - /* Optionally, space(s) can follow. */ while(off < ctx->size && CH(off) == _T(' ')) off++; @@ -4956,11 +5016,13 @@ md_is_opening_code_fence(MD_CTX* ctx, OFF beg, OFF* p_end) while(off < ctx->size && CH(off) == _T(' ')) off++; - /* Optionally, an info string can follow. It must not contain '`'. */ - while(off < ctx->size && CH(off) != _T('`') && !ISNEWLINE(off)) + /* Optionally, an info string can follow. */ + while(off < ctx->size && !ISNEWLINE(off)) { + /* Backtick-based fence must not contain '`' in the info string. */ + if(CH(beg) == _T('`') && CH(off) == _T('`')) + return FALSE; off++; - if(off < ctx->size && !ISNEWLINE(off)) - return FALSE; + } *p_end = off; return TRUE; @@ -5027,7 +5089,7 @@ md_is_html_block_start_condition(MD_CTX* ctx, OFF beg) static const TAG h6[] = { X("h1"), X("head"), X("header"), X("hr"), X("html"), Xend }; static const TAG i6[] = { X("iframe"), Xend }; static const TAG l6[] = { X("legend"), X("li"), X("link"), Xend }; - static const TAG m6[] = { X("main"), X("menu"), X("menuitem"), X("meta"), Xend }; + static const TAG m6[] = { X("main"), X("menu"), X("menuitem"), Xend }; static const TAG n6[] = { X("nav"), X("noframes"), Xend }; static const TAG o6[] = { X("ol"), X("optgroup"), X("option"), Xend }; static const TAG p6[] = { X("p"), X("param"), Xend }; @@ -5047,7 +5109,7 @@ md_is_html_block_start_condition(MD_CTX* ctx, OFF beg) /* Check for type 1: <script, <pre, or <style */ for(i = 0; t1[i].name != NULL; i++) { - if(off + t1[i].len < ctx->size) { + if(off + t1[i].len <= ctx->size) { if(md_ascii_case_eq(STR(off), t1[i].name, t1[i].len)) return 1; } @@ -5328,6 +5390,9 @@ md_is_container_mark(MD_CTX* ctx, unsigned indent, OFF beg, OFF* p_end, MD_CONTA OFF off = beg; OFF max_end; + if(indent >= ctx->code_indent_offset) + return FALSE; + /* Check for block quote mark. */ if(off < ctx->size && CH(off) == _T('>')) { off++; @@ -5406,6 +5471,7 @@ md_analyze_line(MD_CTX* ctx, OFF beg, OFF* p_end, MD_CONTAINER container = { 0 }; int prev_line_has_list_loosening_effect = ctx->last_line_has_list_loosening_effect; OFF off = beg; + OFF hr_killer = 0; int ret = 0; line->indent = md_line_indentation(ctx, total_indent, off, &off); @@ -5441,356 +5507,365 @@ md_analyze_line(MD_CTX* ctx, OFF beg, OFF* p_end, n_parents++; } -redo: - /* Check whether we are fenced code continuation. */ - if(pivot_line->type == MD_LINE_FENCEDCODE) { - line->beg = off; - - /* We are another MD_LINE_FENCEDCODE unless we are closing fence - * which we transform into MD_LINE_BLANK. */ - if(line->indent < ctx->code_indent_offset) { - if(md_is_closing_code_fence(ctx, CH(pivot_line->beg), off, &off)) { - line->type = MD_LINE_BLANK; - ctx->last_line_has_list_loosening_effect = FALSE; - goto done; - } + if(off >= ctx->size || ISNEWLINE(off)) { + /* Blank line does not need any real indentation to be nested inside + * a list. */ + if(n_brothers + n_children == 0) { + while(n_parents < ctx->n_containers && ctx->containers[n_parents].ch != _T('>')) + n_parents++; } + } - if(off >= ctx->size || ISNEWLINE(off)) { - /* Blank line does not need any real indentation to be nested inside - * a list. */ - if(n_brothers + n_children == 0) { - while(n_parents < ctx->n_containers && ctx->containers[n_parents].ch != _T('>')) - n_parents++; + while(TRUE) { + /* Check whether we are fenced code continuation. */ + if(pivot_line->type == MD_LINE_FENCEDCODE) { + line->beg = off; + + /* We are another MD_LINE_FENCEDCODE unless we are closing fence + * which we transform into MD_LINE_BLANK. */ + if(line->indent < ctx->code_indent_offset) { + if(md_is_closing_code_fence(ctx, CH(pivot_line->beg), off, &off)) { + line->type = MD_LINE_BLANK; + ctx->last_line_has_list_loosening_effect = FALSE; + break; + } } - } - /* Change indentation accordingly to the initial code fence. */ - if(n_parents == ctx->n_containers) { - if(line->indent > pivot_line->indent) - line->indent -= pivot_line->indent; - else - line->indent = 0; + /* Change indentation accordingly to the initial code fence. */ + if(n_parents == ctx->n_containers) { + if(line->indent > pivot_line->indent) + line->indent -= pivot_line->indent; + else + line->indent = 0; - line->type = MD_LINE_FENCEDCODE; - goto done; + line->type = MD_LINE_FENCEDCODE; + break; + } } - } - /* Check whether we are HTML block continuation. */ - if(pivot_line->type == MD_LINE_HTML && ctx->html_block_type > 0) { - int html_block_type; + /* Check whether we are HTML block continuation. */ + if(pivot_line->type == MD_LINE_HTML && ctx->html_block_type > 0) { + int html_block_type; - html_block_type = md_is_html_block_end_condition(ctx, off, &off); - if(html_block_type > 0) { - MD_ASSERT(html_block_type == ctx->html_block_type); + html_block_type = md_is_html_block_end_condition(ctx, off, &off); + if(html_block_type > 0) { + MD_ASSERT(html_block_type == ctx->html_block_type); - /* Make sure this is the last line of the block. */ - ctx->html_block_type = 0; + /* Make sure this is the last line of the block. */ + ctx->html_block_type = 0; - /* Some end conditions serve as blank lines at the same time. */ - if(html_block_type == 6 || html_block_type == 7) { - line->type = MD_LINE_BLANK; - line->indent = 0; - goto done; + /* Some end conditions serve as blank lines at the same time. */ + if(html_block_type == 6 || html_block_type == 7) { + line->type = MD_LINE_BLANK; + line->indent = 0; + break; + } } - } - - if(n_parents == ctx->n_containers) { - line->type = MD_LINE_HTML; - goto done; - } - } - - /* Check for blank line. */ - if(off >= ctx->size || ISNEWLINE(off)) { - /* Blank line does not need any real indentation to be nested inside - * a list. */ - if(n_brothers + n_children == 0) { - while(n_parents < ctx->n_containers && ctx->containers[n_parents].ch != _T('>')) - n_parents++; - } - if(pivot_line->type == MD_LINE_INDENTEDCODE && n_parents == ctx->n_containers) { - line->type = MD_LINE_INDENTEDCODE; - if(line->indent > ctx->code_indent_offset) - line->indent -= ctx->code_indent_offset; - else - line->indent = 0; - ctx->last_line_has_list_loosening_effect = FALSE; - } else { - line->type = MD_LINE_BLANK; - ctx->last_line_has_list_loosening_effect = (n_parents > 0 && - n_brothers + n_children == 0 && - ctx->containers[n_parents-1].ch != _T('>')); - -#if 1 - /* See https://github.com/mity/md4c/issues/6 - * - * This ugly checking tests we are in (yet empty) list item but not - * its very first line (with the list item mark). - * - * If we are such blank line, then any following non-blank line - * which would be part of this list item actually ends the list - * because "a list item can begin with at most one blank line." - */ - if(n_parents > 0 && ctx->containers[n_parents-1].ch != _T('>') && - n_brothers + n_children == 0 && ctx->current_block == NULL && - ctx->n_block_bytes > sizeof(MD_BLOCK)) - { - MD_BLOCK* top_block = (MD_BLOCK*) ((char*)ctx->block_bytes + ctx->n_block_bytes - sizeof(MD_BLOCK)); - if(top_block->type == MD_BLOCK_LI) - ctx->last_list_item_starts_with_two_blank_lines = TRUE; + if(n_parents == ctx->n_containers) { + line->type = MD_LINE_HTML; + break; } -#endif } - goto done_on_eol; - } else { -#if 1 - /* This is 2nd half of the hack. If the flag is set (that is there - * were 2nd blank line at the start of the list item) and we would also - * belonging to such list item, then interrupt the list. */ - ctx->last_line_has_list_loosening_effect = FALSE; - if(ctx->last_list_item_starts_with_two_blank_lines) { - if(n_parents > 0 && ctx->containers[n_parents-1].ch != _T('>') && - n_brothers + n_children == 0 && ctx->current_block == NULL && - ctx->n_block_bytes > sizeof(MD_BLOCK)) - { - MD_BLOCK* top_block = (MD_BLOCK*) ((char*)ctx->block_bytes + ctx->n_block_bytes - sizeof(MD_BLOCK)); - if(top_block->type == MD_BLOCK_LI) - n_parents--; + + /* Check for blank line. */ + if(off >= ctx->size || ISNEWLINE(off)) { + if(pivot_line->type == MD_LINE_INDENTEDCODE && n_parents == ctx->n_containers) { + line->type = MD_LINE_INDENTEDCODE; + if(line->indent > ctx->code_indent_offset) + line->indent -= ctx->code_indent_offset; + else + line->indent = 0; + ctx->last_line_has_list_loosening_effect = FALSE; + } else { + line->type = MD_LINE_BLANK; + ctx->last_line_has_list_loosening_effect = (n_parents > 0 && + n_brothers + n_children == 0 && + ctx->containers[n_parents-1].ch != _T('>')); + + #if 1 + /* See https://github.com/mity/md4c/issues/6 + * + * This ugly checking tests we are in (yet empty) list item but not + * its very first line (with the list item mark). + * + * If we are such blank line, then any following non-blank line + * which would be part of this list item actually ends the list + * because "a list item can begin with at most one blank line." + */ + if(n_parents > 0 && ctx->containers[n_parents-1].ch != _T('>') && + n_brothers + n_children == 0 && ctx->current_block == NULL && + ctx->n_block_bytes > (int) sizeof(MD_BLOCK)) + { + MD_BLOCK* top_block = (MD_BLOCK*) ((char*)ctx->block_bytes + ctx->n_block_bytes - sizeof(MD_BLOCK)); + if(top_block->type == MD_BLOCK_LI) + ctx->last_list_item_starts_with_two_blank_lines = TRUE; + } + #endif } + break; + } else { + #if 1 + /* This is 2nd half of the hack. If the flag is set (that is there + * were 2nd blank line at the start of the list item) and we would also + * belonging to such list item, then interrupt the list. */ + ctx->last_line_has_list_loosening_effect = FALSE; + if(ctx->last_list_item_starts_with_two_blank_lines) { + if(n_parents > 0 && ctx->containers[n_parents-1].ch != _T('>') && + n_brothers + n_children == 0 && ctx->current_block == NULL && + ctx->n_block_bytes > (int) sizeof(MD_BLOCK)) + { + MD_BLOCK* top_block = (MD_BLOCK*) ((char*)ctx->block_bytes + ctx->n_block_bytes - sizeof(MD_BLOCK)); + if(top_block->type == MD_BLOCK_LI) + n_parents--; + } - ctx->last_list_item_starts_with_two_blank_lines = FALSE; + ctx->last_list_item_starts_with_two_blank_lines = FALSE; + } + #endif } -#endif - } - /* Check whether we are Setext underline. */ - if(line->indent < ctx->code_indent_offset && pivot_line->type == MD_LINE_TEXT - && (CH(off) == _T('=') || CH(off) == _T('-')) - && (n_parents == ctx->n_containers)) - { - unsigned level; + /* Check whether we are Setext underline. */ + if(line->indent < ctx->code_indent_offset && pivot_line->type == MD_LINE_TEXT + && (CH(off) == _T('=') || CH(off) == _T('-')) + && (n_parents == ctx->n_containers)) + { + unsigned level; - if(md_is_setext_underline(ctx, off, &off, &level)) { - line->type = MD_LINE_SETEXTUNDERLINE; - line->data = level; - goto done; + if(md_is_setext_underline(ctx, off, &off, &level)) { + line->type = MD_LINE_SETEXTUNDERLINE; + line->data = level; + break; + } } - } - /* Check for thematic break line. */ - if(line->indent < ctx->code_indent_offset && ISANYOF(off, _T("-_*"))) { - if(md_is_hr_line(ctx, off, &off)) { - line->type = MD_LINE_HR; - goto done; + /* Check for thematic break line. */ + if(line->indent < ctx->code_indent_offset && ISANYOF(off, _T("-_*")) && off >= hr_killer) { + if(md_is_hr_line(ctx, off, &off, &hr_killer)) { + line->type = MD_LINE_HR; + break; + } } - } - - /* Check for "brother" container. I.e. whether we are another list item - * in already started list. */ - if(n_parents < ctx->n_containers && n_brothers + n_children == 0) { - OFF tmp; - if(md_is_container_mark(ctx, line->indent, off, &tmp, &container) && - md_is_container_compatible(&ctx->containers[n_parents], &container)) - { - pivot_line = &md_dummy_blank_line; + /* Check for "brother" container. I.e. whether we are another list item + * in already started list. */ + if(n_parents < ctx->n_containers && n_brothers + n_children == 0) { + OFF tmp; - off = tmp; + if(md_is_container_mark(ctx, line->indent, off, &tmp, &container) && + md_is_container_compatible(&ctx->containers[n_parents], &container)) + { + pivot_line = &md_dummy_blank_line; - total_indent += container.contents_indent - container.mark_indent; - line->indent = md_line_indentation(ctx, total_indent, off, &off); - total_indent += line->indent; - line->beg = off; + off = tmp; - /* Some of the following whitespace actually still belongs to the mark. */ - if(off >= ctx->size || ISNEWLINE(off)) { - container.contents_indent++; - } else if(line->indent <= ctx->code_indent_offset) { - container.contents_indent += line->indent; - line->indent = 0; - } else { - container.contents_indent += 1; - line->indent--; - } + total_indent += container.contents_indent - container.mark_indent; + line->indent = md_line_indentation(ctx, total_indent, off, &off); + total_indent += line->indent; + line->beg = off; + + /* Some of the following whitespace actually still belongs to the mark. */ + if(off >= ctx->size || ISNEWLINE(off)) { + container.contents_indent++; + } else if(line->indent <= ctx->code_indent_offset) { + container.contents_indent += line->indent; + line->indent = 0; + } else { + container.contents_indent += 1; + line->indent--; + } - ctx->containers[n_parents].mark_indent = container.mark_indent; - ctx->containers[n_parents].contents_indent = container.contents_indent; + ctx->containers[n_parents].mark_indent = container.mark_indent; + ctx->containers[n_parents].contents_indent = container.contents_indent; - n_brothers++; - goto redo; + n_brothers++; + continue; + } } - } - - /* Check for indented code. - * Note indented code block cannot interrupt a paragraph. */ - if(line->indent >= ctx->code_indent_offset && - (pivot_line->type == MD_LINE_BLANK || pivot_line->type == MD_LINE_INDENTEDCODE)) - { - line->type = MD_LINE_INDENTEDCODE; - MD_ASSERT(line->indent >= ctx->code_indent_offset); - line->indent -= ctx->code_indent_offset; - line->data = 0; - goto done; - } - /* Check for start of a new container block. */ - if(line->indent < ctx->code_indent_offset && - md_is_container_mark(ctx, line->indent, off, &off, &container)) - { - if(pivot_line->type == MD_LINE_TEXT && n_parents == ctx->n_containers && - (off >= ctx->size || ISNEWLINE(off))) + /* Check for indented code. + * Note indented code block cannot interrupt a paragraph. */ + if(line->indent >= ctx->code_indent_offset && + (pivot_line->type == MD_LINE_BLANK || pivot_line->type == MD_LINE_INDENTEDCODE)) { - /* Noop. List mark followed by a blank line cannot interrupt a paragraph. */ - } else if(pivot_line->type == MD_LINE_TEXT && n_parents == ctx->n_containers && - (container.ch == _T('.') || container.ch == _T(')')) && container.start != 1) - { - /* Noop. Ordered list cannot interrupt a paragraph unless the start index is 1. */ - } else { - total_indent += container.contents_indent - container.mark_indent; - line->indent = md_line_indentation(ctx, total_indent, off, &off); - total_indent += line->indent; + line->type = MD_LINE_INDENTEDCODE; + MD_ASSERT(line->indent >= ctx->code_indent_offset); + line->indent -= ctx->code_indent_offset; + line->data = 0; + break; + } - line->beg = off; - line->data = container.ch; - - /* Some of the following whitespace actually still belongs to the mark. */ - if(off >= ctx->size || ISNEWLINE(off)) { - container.contents_indent++; - } else if(line->indent <= ctx->code_indent_offset) { - container.contents_indent += line->indent; - line->indent = 0; + /* Check for start of a new container block. */ + if(line->indent < ctx->code_indent_offset && + md_is_container_mark(ctx, line->indent, off, &off, &container)) + { + if(pivot_line->type == MD_LINE_TEXT && n_parents == ctx->n_containers && + (off >= ctx->size || ISNEWLINE(off))) + { + /* Noop. List mark followed by a blank line cannot interrupt a paragraph. */ + } else if(pivot_line->type == MD_LINE_TEXT && n_parents == ctx->n_containers && + (container.ch == _T('.') || container.ch == _T(')')) && container.start != 1) + { + /* Noop. Ordered list cannot interrupt a paragraph unless the start index is 1. */ } else { - container.contents_indent += 1; - line->indent--; - } + total_indent += container.contents_indent - container.mark_indent; + line->indent = md_line_indentation(ctx, total_indent, off, &off); + total_indent += line->indent; + + line->beg = off; + line->data = container.ch; + + /* Some of the following whitespace actually still belongs to the mark. */ + if(off >= ctx->size || ISNEWLINE(off)) { + container.contents_indent++; + } else if(line->indent <= ctx->code_indent_offset) { + container.contents_indent += line->indent; + line->indent = 0; + } else { + container.contents_indent += 1; + line->indent--; + } - if(n_brothers + n_children == 0) - pivot_line = &md_dummy_blank_line; + if(n_brothers + n_children == 0) + pivot_line = &md_dummy_blank_line; - if(n_children == 0) - MD_CHECK(md_leave_child_containers(ctx, n_parents + n_brothers)); + if(n_children == 0) + MD_CHECK(md_leave_child_containers(ctx, n_parents + n_brothers)); - n_children++; - MD_CHECK(md_push_container(ctx, &container)); - goto redo; + n_children++; + MD_CHECK(md_push_container(ctx, &container)); + continue; + } } - } - /* Check whether we are table continuation. */ - if(pivot_line->type == MD_LINE_TABLE && md_is_table_row(ctx, off, &off) && - n_parents == ctx->n_containers) - { - line->type = MD_LINE_TABLE; - goto done; - } + /* Check whether we are table continuation. */ + if(pivot_line->type == MD_LINE_TABLE && md_is_table_row(ctx, off, &off) && + n_parents == ctx->n_containers) + { + line->type = MD_LINE_TABLE; + break; + } - /* Check for ATX header. */ - if(line->indent < ctx->code_indent_offset && CH(off) == _T('#')) { - unsigned level; + /* Check for ATX header. */ + if(line->indent < ctx->code_indent_offset && CH(off) == _T('#')) { + unsigned level; - if(md_is_atxheader_line(ctx, off, &line->beg, &off, &level)) { - line->type = MD_LINE_ATXHEADER; - line->data = level; - goto done; + if(md_is_atxheader_line(ctx, off, &line->beg, &off, &level)) { + line->type = MD_LINE_ATXHEADER; + line->data = level; + break; + } } - } - /* Check whether we are starting code fence. */ - if(CH(off) == _T('`') || CH(off) == _T('~')) { - if(md_is_opening_code_fence(ctx, off, &off)) { - line->type = MD_LINE_FENCEDCODE; - line->data = 1; - goto done; + /* Check whether we are starting code fence. */ + if(CH(off) == _T('`') || CH(off) == _T('~')) { + if(md_is_opening_code_fence(ctx, off, &off)) { + line->type = MD_LINE_FENCEDCODE; + line->data = 1; + break; + } } - } - /* Check for start of raw HTML block. */ - if(CH(off) == _T('<') && !(ctx->parser.flags & MD_FLAG_NOHTMLBLOCKS)) - { - ctx->html_block_type = md_is_html_block_start_condition(ctx, off); - - /* HTML block type 7 cannot interrupt paragraph. */ - if(ctx->html_block_type == 7 && pivot_line->type == MD_LINE_TEXT) - ctx->html_block_type = 0; + /* Check for start of raw HTML block. */ + if(CH(off) == _T('<') && !(ctx->parser.flags & MD_FLAG_NOHTMLBLOCKS)) + { + ctx->html_block_type = md_is_html_block_start_condition(ctx, off); - if(ctx->html_block_type > 0) { - /* The line itself also may immediately close the block. */ - if(md_is_html_block_end_condition(ctx, off, &off) == ctx->html_block_type) { - /* Make sure this is the last line of the block. */ + /* HTML block type 7 cannot interrupt paragraph. */ + if(ctx->html_block_type == 7 && pivot_line->type == MD_LINE_TEXT) ctx->html_block_type = 0; - } - line->type = MD_LINE_HTML; - goto done; - } - } + if(ctx->html_block_type > 0) { + /* The line itself also may immediately close the block. */ + if(md_is_html_block_end_condition(ctx, off, &off) == ctx->html_block_type) { + /* Make sure this is the last line of the block. */ + ctx->html_block_type = 0; + } - /* Check for table underline. */ - if((ctx->parser.flags & MD_FLAG_TABLES) && pivot_line->type == MD_LINE_TEXT && - (CH(off) == _T('|') || CH(off) == _T('-') || CH(off) == _T(':')) && - n_parents == ctx->n_containers) - { - unsigned col_count; + line->type = MD_LINE_HTML; + break; + } + } - if(ctx->current_block != NULL && ctx->current_block->n_lines == 1 && - md_is_table_underline(ctx, off, &off, &col_count) && - md_is_table_row(ctx, pivot_line->beg, NULL)) + /* Check for table underline. */ + if((ctx->parser.flags & MD_FLAG_TABLES) && pivot_line->type == MD_LINE_TEXT && + (CH(off) == _T('|') || CH(off) == _T('-') || CH(off) == _T(':')) && + n_parents == ctx->n_containers) { - line->data = col_count; - line->type = MD_LINE_TABLEUNDERLINE; - goto done; - } - } + unsigned col_count; - /* By default, we are normal text line. */ - line->type = MD_LINE_TEXT; - if(pivot_line->type == MD_LINE_TEXT && n_brothers + n_children == 0) { - /* Lazy continuation. */ - n_parents = ctx->n_containers; - } + if(ctx->current_block != NULL && ctx->current_block->n_lines == 1 && + md_is_table_underline(ctx, off, &off, &col_count) && + md_is_table_row(ctx, pivot_line->beg, NULL)) + { + line->data = col_count; + line->type = MD_LINE_TABLEUNDERLINE; + break; + } + } - /* Check for task mark. */ - if((ctx->parser.flags & MD_FLAG_TASKLISTS) && n_brothers + n_children > 0 && - ISANYOF_(ctx->containers[ctx->n_containers-1].ch, _T("-+*.)"))) - { - OFF tmp = off; + /* By default, we are normal text line. */ + line->type = MD_LINE_TEXT; + if(pivot_line->type == MD_LINE_TEXT && n_brothers + n_children == 0) { + /* Lazy continuation. */ + n_parents = ctx->n_containers; + } - while(tmp < ctx->size && tmp < off + 3 && ISBLANK(tmp)) - tmp++; - if(tmp + 2 < ctx->size && CH(tmp) == _T('[') && - ISANYOF(tmp+1, _T("xX ")) && CH(tmp+2) == _T(']') && - (tmp + 3 == ctx->size || ISBLANK(tmp+3) || ISNEWLINE(tmp+3))) + /* Check for task mark. */ + if((ctx->parser.flags & MD_FLAG_TASKLISTS) && n_brothers + n_children > 0 && + ISANYOF_(ctx->containers[ctx->n_containers-1].ch, _T("-+*.)"))) { - MD_CONTAINER* task_container = (n_children > 0 ? &ctx->containers[ctx->n_containers-1] : &container); - task_container->is_task = TRUE; - task_container->task_mark_off = tmp + 1; - off = tmp + 3; - while(ISWHITESPACE(off)) - off++; - line->beg = off; + OFF tmp = off; + + while(tmp < ctx->size && tmp < off + 3 && ISBLANK(tmp)) + tmp++; + if(tmp + 2 < ctx->size && CH(tmp) == _T('[') && + ISANYOF(tmp+1, _T("xX ")) && CH(tmp+2) == _T(']') && + (tmp + 3 == ctx->size || ISBLANK(tmp+3) || ISNEWLINE(tmp+3))) + { + MD_CONTAINER* task_container = (n_children > 0 ? &ctx->containers[ctx->n_containers-1] : &container); + task_container->is_task = TRUE; + task_container->task_mark_off = tmp + 1; + off = tmp + 3; + while(ISWHITESPACE(off)) + off++; + line->beg = off; + } } + + break; } -done: /* Scan for end of the line. * - * Note this is bottleneck of this function as we itereate over (almost) - * all line contents after some initial line indentation. To optimize, we - * try to eat multiple chars in every loop iteration. - * - * (Measured ~6% performance boost of md2html with this optimization for - * normal kind of input.) + * Note this is quite a bottleneck of the parsing as we here iterate almost + * over compete document. */ - while(off + 4 < ctx->size && !ISNEWLINE(off+0) && !ISNEWLINE(off+1) - && !ISNEWLINE(off+2) && !ISNEWLINE(off+3)) - off += 4; - while(off < ctx->size && !ISNEWLINE(off)) - off++; +#if defined __linux__ && !defined MD4C_USE_UTF16 + /* Recent glibc versions have superbly optimized strcspn(), even using + * vectorization if available. */ + if(ctx->doc_ends_with_newline && off < ctx->size) { + while(TRUE) { + off += (OFF) strcspn(STR(off), "\r\n"); + + /* strcspn() can stop on zero terminator; but that can appear + * anywhere in the Markfown input... */ + if(CH(off) == _T('\0')) + off++; + else + break; + } + } else +#endif + { + /* Optimization: Use some loop unrolling. */ + while(off + 3 < ctx->size && !ISNEWLINE(off+0) && !ISNEWLINE(off+1) + && !ISNEWLINE(off+2) && !ISNEWLINE(off+3)) + off += 4; + while(off < ctx->size && !ISNEWLINE(off)) + off++; + } -done_on_eol: /* Set end of the line. */ line->end = off; @@ -5855,7 +5930,7 @@ abort: } static int -md_process_line(MD_CTX* ctx, const MD_LINE_ANALYSIS** p_pivot_line, const MD_LINE_ANALYSIS* line) +md_process_line(MD_CTX* ctx, const MD_LINE_ANALYSIS** p_pivot_line, MD_LINE_ANALYSIS* line) { const MD_LINE_ANALYSIS* pivot_line = *p_pivot_line; int ret = 0; @@ -5884,8 +5959,17 @@ md_process_line(MD_CTX* ctx, const MD_LINE_ANALYSIS** p_pivot_line, const MD_LIN MD_ASSERT(ctx->current_block != NULL); ctx->current_block->type = MD_BLOCK_H; ctx->current_block->data = line->data; + ctx->current_block->flags |= MD_BLOCK_SETEXT_HEADER; + MD_CHECK(md_add_line_into_current_block(ctx, line)); MD_CHECK(md_end_current_block(ctx)); - *p_pivot_line = &md_dummy_blank_line; + if(ctx->current_block == NULL) { + *p_pivot_line = &md_dummy_blank_line; + } else { + /* This happens if we have consumed all the body as link ref. defs. + * and downgraded the underline into start of a new paragraph block. */ + line->type = MD_LINE_TEXT; + *p_pivot_line = line; + } return 0; } @@ -6000,9 +6084,10 @@ md_parse(const MD_CHAR* text, MD_SIZE size, const MD_PARSER* parser, void* userd ctx.userdata = userdata; ctx.code_indent_offset = (ctx.parser.flags & MD_FLAG_NOINDENTEDCODEBLOCKS) ? (OFF)(-1) : 4; md_build_mark_char_map(&ctx); + ctx.doc_ends_with_newline = (size > 0 && ISNEWLINE_(text[size-1])); /* Reset all unresolved opener mark chains. */ - for(i = 0; i < SIZEOF_ARRAY(ctx.mark_chains); i++) { + for(i = 0; i < (int) SIZEOF_ARRAY(ctx.mark_chains); i++) { ctx.mark_chains[i].head = -1; ctx.mark_chains[i].tail = -1; } diff --git a/src/3rdparty/md4c/md4c.h b/src/3rdparty/md4c/md4c.h index 10cba67e95..dcdadad88d 100644 --- a/src/3rdparty/md4c/md4c.h +++ b/src/3rdparty/md4c/md4c.h @@ -30,10 +30,12 @@ extern "C" { #endif -/* Magic to support UTF-16. */ #if defined MD4C_USE_UTF16 + /* Magic to support UTF-16. Not that in order to use it, you have to define + * the macro MD4C_USE_UTF16 both when building MD4C as well as when + * including this header in your code. */ #ifdef _WIN32 - #include <wchar.h> + #include <windows.h> typedef WCHAR MD_CHAR; #else #error MD4C_USE_UTF16 is only supported on Windows. @@ -236,6 +238,7 @@ typedef struct MD_BLOCK_H_DETAIL { typedef struct MD_BLOCK_CODE_DETAIL { MD_ATTRIBUTE info; MD_ATTRIBUTE lang; + MD_CHAR fence_char; /* The character used for fenced code block; or zero for indented code block. */ } MD_BLOCK_CODE_DETAIL; /* Detailed info for MD_BLOCK_TH and MD_BLOCK_TD. */ diff --git a/src/3rdparty/md4c/qt_attribution.json b/src/3rdparty/md4c/qt_attribution.json index 9180ed69b5..fa0fd18853 100644 --- a/src/3rdparty/md4c/qt_attribution.json +++ b/src/3rdparty/md4c/qt_attribution.json @@ -9,7 +9,7 @@ "License": "MIT License", "LicenseId": "MIT", "LicenseFile": "LICENSE.md", - "Version": "0.3.0", - "DownloadLocation": "https://github.com/mity/md4c/releases/tag/release-0.3.0-rc", + "Version": "0.3.3", + "DownloadLocation": "https://github.com/mity/md4c/releases/tag/release-0.3.3", "Copyright": "Copyright © 2016-2019 Martin Mitáš" } |