11 // The following notes assume that you are familiar with the YAML specification
12 // (http://yaml.org/spec/1.2/spec.html). We mostly follow it, although in
13 // some cases we are less restrictive that it requires.
15 // The process of transforming a YAML stream into a sequence of events is
16 // divided on two steps: Scanning and Parsing.
18 // The Scanner transforms the input stream into a sequence of tokens, while the
19 // parser transform the sequence of tokens produced by the Scanner into a
20 // sequence of parsing events.
22 // The Scanner is rather clever and complicated. The Parser, on the contrary,
23 // is a straightforward implementation of a recursive-descendant parser (or,
24 // LL(1) parser, as it is usually called).
26 // Actually there are two issues of Scanning that might be called "clever", the
27 // rest is quite straightforward. The issues are "block collection start" and
28 // "simple keys". Both issues are explained below in details.
30 // Here the Scanning step is explained and implemented. We start with the list
31 // of all the tokens produced by the Scanner together with short descriptions.
35 // STREAM-START(encoding) # The stream start.
36 // STREAM-END # The stream end.
37 // VERSION-DIRECTIVE(major,minor) # The '%YAML' directive.
38 // TAG-DIRECTIVE(handle,prefix) # The '%TAG' directive.
39 // DOCUMENT-START # '---'
40 // DOCUMENT-END # '...'
41 // BLOCK-SEQUENCE-START # Indentation increase denoting a block
42 // BLOCK-MAPPING-START # sequence or a block mapping.
43 // BLOCK-END # Indentation decrease.
44 // FLOW-SEQUENCE-START # '['
45 // FLOW-SEQUENCE-END # ']'
46 // BLOCK-SEQUENCE-START # '{'
47 // BLOCK-SEQUENCE-END # '}'
50 // KEY # '?' or nothing (simple keys).
52 // ALIAS(anchor) # '*anchor'
53 // ANCHOR(anchor) # '&anchor'
54 // TAG(handle,suffix) # '!handle!suffix'
55 // SCALAR(value,style) # A scalar.
57 // The following two tokens are "virtual" tokens denoting the beginning and the
60 // STREAM-START(encoding)
63 // We pass the information about the input stream encoding with the
64 // STREAM-START token.
66 // The next two tokens are responsible for tags:
68 // VERSION-DIRECTIVE(major,minor)
69 // TAG-DIRECTIVE(handle,prefix)
75 // %TAG !yaml! tag:yaml.org,2002:
78 // The correspoding sequence of tokens:
80 // STREAM-START(utf-8)
81 // VERSION-DIRECTIVE(1,1)
82 // TAG-DIRECTIVE("!","!foo")
83 // TAG-DIRECTIVE("!yaml","tag:yaml.org,2002:")
87 // Note that the VERSION-DIRECTIVE and TAG-DIRECTIVE tokens occupy a whole
90 // The document start and end indicators are represented by:
95 // Note that if a YAML stream contains an implicit document (without '---'
96 // and '...' indicators), no DOCUMENT-START and DOCUMENT-END tokens will be
99 // In the following examples, we present whole documents together with the
102 // 1. An implicit document:
108 // STREAM-START(utf-8)
109 // SCALAR("a scalar",single-quoted)
112 // 2. An explicit document:
120 // STREAM-START(utf-8)
122 // SCALAR("a scalar",single-quoted)
126 // 3. Several documents in a stream:
132 // 'yet another scalar'
136 // STREAM-START(utf-8)
137 // SCALAR("a scalar",single-quoted)
139 // SCALAR("another scalar",single-quoted)
141 // SCALAR("yet another scalar",single-quoted)
144 // We have already introduced the SCALAR token above. The following tokens are
145 // used to describe aliases, anchors, tag, and scalars:
149 // TAG(handle,suffix)
150 // SCALAR(value,style)
152 // The following series of examples illustrate the usage of these tokens:
154 // 1. A recursive sequence:
160 // STREAM-START(utf-8)
162 // FLOW-SEQUENCE-START
167 // 2. A tagged scalar:
169 // !!float "3.14" # A good approximation.
173 // STREAM-START(utf-8)
175 // SCALAR("3.14",double-quoted)
178 // 3. Various scalar styles:
180 // --- # Implicit empty plain scalars do not produce tokens.
181 // --- a plain scalar
182 // --- 'a single-quoted scalar'
183 // --- "a double-quoted scalar"
192 // STREAM-START(utf-8)
195 // SCALAR("a plain scalar",plain)
197 // SCALAR("a single-quoted scalar",single-quoted)
199 // SCALAR("a double-quoted scalar",double-quoted)
201 // SCALAR("a literal scalar",literal)
203 // SCALAR("a folded scalar",folded)
206 // Now it's time to review collection-related tokens. We will start with
209 // FLOW-SEQUENCE-START
211 // FLOW-MAPPING-START
217 // The tokens FLOW-SEQUENCE-START, FLOW-SEQUENCE-END, FLOW-MAPPING-START, and
218 // FLOW-MAPPING-END represent the indicators '[', ']', '{', and '}'
219 // correspondingly. FLOW-ENTRY represent the ',' indicator. Finally the
220 // indicators '?' and ':', which are used for denoting mapping keys and values,
221 // are represented by the KEY and VALUE tokens.
223 // The following examples show flow collections:
225 // 1. A flow sequence:
227 // [item 1, item 2, item 3]
231 // STREAM-START(utf-8)
232 // FLOW-SEQUENCE-START
233 // SCALAR("item 1",plain)
235 // SCALAR("item 2",plain)
237 // SCALAR("item 3",plain)
241 // 2. A flow mapping:
244 // a simple key: a value, # Note that the KEY token is produced.
245 // ? a complex key: another value,
250 // STREAM-START(utf-8)
251 // FLOW-MAPPING-START
253 // SCALAR("a simple key",plain)
255 // SCALAR("a value",plain)
258 // SCALAR("a complex key",plain)
260 // SCALAR("another value",plain)
265 // A simple key is a key which is not denoted by the '?' indicator. Note that
266 // the Scanner still produce the KEY token whenever it encounters a simple key.
268 // For scanning block collections, the following tokens are used (note that we
269 // repeat KEY and VALUE here):
271 // BLOCK-SEQUENCE-START
272 // BLOCK-MAPPING-START
278 // The tokens BLOCK-SEQUENCE-START and BLOCK-MAPPING-START denote indentation
279 // increase that precedes a block collection (cf. the INDENT token in Python).
280 // The token BLOCK-END denote indentation decrease that ends a block collection
281 // (cf. the DEDENT token in Python). However YAML has some syntax pecularities
282 // that makes detections of these tokens more complex.
284 // The tokens BLOCK-ENTRY, KEY, and VALUE are used to represent the indicators
285 // '-', '?', and ':' correspondingly.
287 // The following examples show how the tokens BLOCK-SEQUENCE-START,
288 // BLOCK-MAPPING-START, and BLOCK-END are emitted by the Scanner:
290 // 1. Block sequences:
303 // STREAM-START(utf-8)
304 // BLOCK-SEQUENCE-START
306 // SCALAR("item 1",plain)
308 // SCALAR("item 2",plain)
310 // BLOCK-SEQUENCE-START
312 // SCALAR("item 3.1",plain)
314 // SCALAR("item 3.2",plain)
317 // BLOCK-MAPPING-START
319 // SCALAR("key 1",plain)
321 // SCALAR("value 1",plain)
323 // SCALAR("key 2",plain)
325 // SCALAR("value 2",plain)
330 // 2. Block mappings:
332 // a simple key: a value # The KEY token is produced here.
344 // STREAM-START(utf-8)
345 // BLOCK-MAPPING-START
347 // SCALAR("a simple key",plain)
349 // SCALAR("a value",plain)
351 // SCALAR("a complex key",plain)
353 // SCALAR("another value",plain)
355 // SCALAR("a mapping",plain)
356 // BLOCK-MAPPING-START
358 // SCALAR("key 1",plain)
360 // SCALAR("value 1",plain)
362 // SCALAR("key 2",plain)
364 // SCALAR("value 2",plain)
367 // SCALAR("a sequence",plain)
369 // BLOCK-SEQUENCE-START
371 // SCALAR("item 1",plain)
373 // SCALAR("item 2",plain)
378 // YAML does not always require to start a new block collection from a new
379 // line. If the current line contains only '-', '?', and ':' indicators, a new
380 // block collection may start at the current line. The following examples
381 // illustrate this case:
383 // 1. Collections in a sequence:
394 // STREAM-START(utf-8)
395 // BLOCK-SEQUENCE-START
397 // BLOCK-SEQUENCE-START
399 // SCALAR("item 1",plain)
401 // SCALAR("item 2",plain)
404 // BLOCK-MAPPING-START
406 // SCALAR("key 1",plain)
408 // SCALAR("value 1",plain)
410 // SCALAR("key 2",plain)
412 // SCALAR("value 2",plain)
415 // BLOCK-MAPPING-START
417 // SCALAR("complex key")
419 // SCALAR("complex value")
424 // 2. Collections in a mapping:
435 // STREAM-START(utf-8)
436 // BLOCK-MAPPING-START
438 // SCALAR("a sequence",plain)
440 // BLOCK-SEQUENCE-START
442 // SCALAR("item 1",plain)
444 // SCALAR("item 2",plain)
447 // SCALAR("a mapping",plain)
449 // BLOCK-MAPPING-START
451 // SCALAR("key 1",plain)
453 // SCALAR("value 1",plain)
455 // SCALAR("key 2",plain)
457 // SCALAR("value 2",plain)
462 // YAML also permits non-indented sequences if they are included into a block
463 // mapping. In this case, the token BLOCK-SEQUENCE-START is not produced:
466 // - item 1 # BLOCK-SEQUENCE-START is NOT produced here.
471 // STREAM-START(utf-8)
472 // BLOCK-MAPPING-START
474 // SCALAR("key",plain)
477 // SCALAR("item 1",plain)
479 // SCALAR("item 2",plain)
483 // Ensure that the buffer contains the required number of characters.
484 // Return true on success, false on failure (reader error or memory error).
485 func cache(parser *yaml_parser_t, length int) bool {
486 // [Go] This was inlined: !cache(A, B) -> unread < B && !update(A, B)
487 return parser.unread >= length || yaml_parser_update_buffer(parser, length)
490 // Advance the buffer pointer.
491 func skip(parser *yaml_parser_t) {
495 parser.buffer_pos += width(parser.buffer[parser.buffer_pos])
498 func skip_line(parser *yaml_parser_t) {
499 if is_crlf(parser.buffer, parser.buffer_pos) {
500 parser.mark.index += 2
501 parser.mark.column = 0
504 parser.buffer_pos += 2
505 } else if is_break(parser.buffer, parser.buffer_pos) {
507 parser.mark.column = 0
510 parser.buffer_pos += width(parser.buffer[parser.buffer_pos])
514 // Copy a character to a string buffer and advance pointers.
515 func read(parser *yaml_parser_t, s []byte) []byte {
516 w := width(parser.buffer[parser.buffer_pos])
518 panic("invalid character sequence")
521 s = make([]byte, 0, 32)
523 if w == 1 && len(s)+w <= cap(s) {
525 s[len(s)-1] = parser.buffer[parser.buffer_pos]
528 s = append(s, parser.buffer[parser.buffer_pos:parser.buffer_pos+w]...)
529 parser.buffer_pos += w
537 // Copy a line break character to a string buffer and advance pointers.
538 func read_line(parser *yaml_parser_t, s []byte) []byte {
540 pos := parser.buffer_pos
542 case buf[pos] == '\r' && buf[pos+1] == '\n':
545 parser.buffer_pos += 2
548 case buf[pos] == '\r' || buf[pos] == '\n':
551 parser.buffer_pos += 1
552 case buf[pos] == '\xC2' && buf[pos+1] == '\x85':
555 parser.buffer_pos += 2
556 case buf[pos] == '\xE2' && buf[pos+1] == '\x80' && (buf[pos+2] == '\xA8' || buf[pos+2] == '\xA9'):
558 s = append(s, buf[parser.buffer_pos:pos+3]...)
559 parser.buffer_pos += 3
564 parser.mark.column = 0
570 // Get the next token.
571 func yaml_parser_scan(parser *yaml_parser_t, token *yaml_token_t) bool {
572 // Erase the token object.
573 *token = yaml_token_t{} // [Go] Is this necessary?
575 // No tokens after STREAM-END or error.
576 if parser.stream_end_produced || parser.error != yaml_NO_ERROR {
580 // Ensure that the tokens queue contains enough tokens.
581 if !parser.token_available {
582 if !yaml_parser_fetch_more_tokens(parser) {
587 // Fetch the next token from the queue.
588 *token = parser.tokens[parser.tokens_head]
590 parser.tokens_parsed++
591 parser.token_available = false
593 if token.typ == yaml_STREAM_END_TOKEN {
594 parser.stream_end_produced = true
599 // Set the scanner error and return false.
600 func yaml_parser_set_scanner_error(parser *yaml_parser_t, context string, context_mark yaml_mark_t, problem string) bool {
601 parser.error = yaml_SCANNER_ERROR
602 parser.context = context
603 parser.context_mark = context_mark
604 parser.problem = problem
605 parser.problem_mark = parser.mark
609 func yaml_parser_set_scanner_tag_error(parser *yaml_parser_t, directive bool, context_mark yaml_mark_t, problem string) bool {
610 context := "while parsing a tag"
612 context = "while parsing a %TAG directive"
614 return yaml_parser_set_scanner_error(parser, context, context_mark, problem)
617 func trace(args ...interface{}) func() {
618 pargs := append([]interface{}{"+++"}, args...)
619 fmt.Println(pargs...)
620 pargs = append([]interface{}{"---"}, args...)
621 return func() { fmt.Println(pargs...) }
624 // Ensure that the tokens queue contains at least one token which can be
625 // returned to the Parser.
626 func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool {
627 // While we need more tokens to fetch, do it.
629 // Check if we really need to fetch more tokens.
630 need_more_tokens := false
632 if parser.tokens_head == len(parser.tokens) {
634 need_more_tokens = true
636 // Check if any potential simple key may occupy the head position.
637 if !yaml_parser_stale_simple_keys(parser) {
641 for i := range parser.simple_keys {
642 simple_key := &parser.simple_keys[i]
643 if simple_key.possible && simple_key.token_number == parser.tokens_parsed {
644 need_more_tokens = true
651 if !need_more_tokens {
654 // Fetch the next token.
655 if !yaml_parser_fetch_next_token(parser) {
660 parser.token_available = true
664 // The dispatcher for token fetchers.
665 func yaml_parser_fetch_next_token(parser *yaml_parser_t) bool {
666 // Ensure that the buffer is initialized.
667 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
671 // Check if we just started scanning. Fetch STREAM-START then.
672 if !parser.stream_start_produced {
673 return yaml_parser_fetch_stream_start(parser)
676 // Eat whitespaces and comments until we reach the next token.
677 if !yaml_parser_scan_to_next_token(parser) {
681 // Remove obsolete potential simple keys.
682 if !yaml_parser_stale_simple_keys(parser) {
686 // Check the indentation level against the current column.
687 if !yaml_parser_unroll_indent(parser, parser.mark.column) {
691 // Ensure that the buffer contains at least 4 characters. 4 is the length
692 // of the longest indicators ('--- ' and '... ').
693 if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) {
697 // Is it the end of the stream?
698 if is_z(parser.buffer, parser.buffer_pos) {
699 return yaml_parser_fetch_stream_end(parser)
702 // Is it a directive?
703 if parser.mark.column == 0 && parser.buffer[parser.buffer_pos] == '%' {
704 return yaml_parser_fetch_directive(parser)
708 pos := parser.buffer_pos
710 // Is it the document start indicator?
711 if parser.mark.column == 0 && buf[pos] == '-' && buf[pos+1] == '-' && buf[pos+2] == '-' && is_blankz(buf, pos+3) {
712 return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_START_TOKEN)
715 // Is it the document end indicator?
716 if parser.mark.column == 0 && buf[pos] == '.' && buf[pos+1] == '.' && buf[pos+2] == '.' && is_blankz(buf, pos+3) {
717 return yaml_parser_fetch_document_indicator(parser, yaml_DOCUMENT_END_TOKEN)
720 // Is it the flow sequence start indicator?
722 return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_SEQUENCE_START_TOKEN)
725 // Is it the flow mapping start indicator?
726 if parser.buffer[parser.buffer_pos] == '{' {
727 return yaml_parser_fetch_flow_collection_start(parser, yaml_FLOW_MAPPING_START_TOKEN)
730 // Is it the flow sequence end indicator?
731 if parser.buffer[parser.buffer_pos] == ']' {
732 return yaml_parser_fetch_flow_collection_end(parser,
733 yaml_FLOW_SEQUENCE_END_TOKEN)
736 // Is it the flow mapping end indicator?
737 if parser.buffer[parser.buffer_pos] == '}' {
738 return yaml_parser_fetch_flow_collection_end(parser,
739 yaml_FLOW_MAPPING_END_TOKEN)
742 // Is it the flow entry indicator?
743 if parser.buffer[parser.buffer_pos] == ',' {
744 return yaml_parser_fetch_flow_entry(parser)
747 // Is it the block entry indicator?
748 if parser.buffer[parser.buffer_pos] == '-' && is_blankz(parser.buffer, parser.buffer_pos+1) {
749 return yaml_parser_fetch_block_entry(parser)
752 // Is it the key indicator?
753 if parser.buffer[parser.buffer_pos] == '?' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) {
754 return yaml_parser_fetch_key(parser)
757 // Is it the value indicator?
758 if parser.buffer[parser.buffer_pos] == ':' && (parser.flow_level > 0 || is_blankz(parser.buffer, parser.buffer_pos+1)) {
759 return yaml_parser_fetch_value(parser)
763 if parser.buffer[parser.buffer_pos] == '*' {
764 return yaml_parser_fetch_anchor(parser, yaml_ALIAS_TOKEN)
768 if parser.buffer[parser.buffer_pos] == '&' {
769 return yaml_parser_fetch_anchor(parser, yaml_ANCHOR_TOKEN)
773 if parser.buffer[parser.buffer_pos] == '!' {
774 return yaml_parser_fetch_tag(parser)
777 // Is it a literal scalar?
778 if parser.buffer[parser.buffer_pos] == '|' && parser.flow_level == 0 {
779 return yaml_parser_fetch_block_scalar(parser, true)
782 // Is it a folded scalar?
783 if parser.buffer[parser.buffer_pos] == '>' && parser.flow_level == 0 {
784 return yaml_parser_fetch_block_scalar(parser, false)
787 // Is it a single-quoted scalar?
788 if parser.buffer[parser.buffer_pos] == '\'' {
789 return yaml_parser_fetch_flow_scalar(parser, true)
792 // Is it a double-quoted scalar?
793 if parser.buffer[parser.buffer_pos] == '"' {
794 return yaml_parser_fetch_flow_scalar(parser, false)
797 // Is it a plain scalar?
799 // A plain scalar may start with any non-blank characters except
801 // '-', '?', ':', ',', '[', ']', '{', '}',
802 // '#', '&', '*', '!', '|', '>', '\'', '\"',
805 // In the block context (and, for the '-' indicator, in the flow context
806 // too), it may also start with the characters
810 // if it is followed by a non-space character.
812 // The last rule is more restrictive than the specification requires.
813 // [Go] Make this logic more reasonable.
814 //switch parser.buffer[parser.buffer_pos] {
815 //case '-', '?', ':', ',', '?', '-', ',', ':', ']', '[', '}', '{', '&', '#', '!', '*', '>', '|', '"', '\'', '@', '%', '-', '`':
817 if !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '-' ||
818 parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':' ||
819 parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '[' ||
820 parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' ||
821 parser.buffer[parser.buffer_pos] == '}' || parser.buffer[parser.buffer_pos] == '#' ||
822 parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '*' ||
823 parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '|' ||
824 parser.buffer[parser.buffer_pos] == '>' || parser.buffer[parser.buffer_pos] == '\'' ||
825 parser.buffer[parser.buffer_pos] == '"' || parser.buffer[parser.buffer_pos] == '%' ||
826 parser.buffer[parser.buffer_pos] == '@' || parser.buffer[parser.buffer_pos] == '`') ||
827 (parser.buffer[parser.buffer_pos] == '-' && !is_blank(parser.buffer, parser.buffer_pos+1)) ||
828 (parser.flow_level == 0 &&
829 (parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == ':') &&
830 !is_blankz(parser.buffer, parser.buffer_pos+1)) {
831 return yaml_parser_fetch_plain_scalar(parser)
834 // If we don't determine the token type so far, it is an error.
835 return yaml_parser_set_scanner_error(parser,
836 "while scanning for the next token", parser.mark,
837 "found character that cannot start any token")
840 // Check the list of potential simple keys and remove the positions that
841 // cannot contain simple keys anymore.
842 func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool {
843 // Check for a potential simple key for each flow level.
844 for i := range parser.simple_keys {
845 simple_key := &parser.simple_keys[i]
847 // The specification requires that a simple key
849 // - is limited to a single line,
850 // - is shorter than 1024 characters.
851 if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) {
853 // Check if the potential simple key to be removed is required.
854 if simple_key.required {
855 return yaml_parser_set_scanner_error(parser,
856 "while scanning a simple key", simple_key.mark,
857 "could not find expected ':'")
859 simple_key.possible = false
865 // Check if a simple key may start at the current position and add it if
867 func yaml_parser_save_simple_key(parser *yaml_parser_t) bool {
868 // A simple key is required at the current position if the scanner is in
869 // the block context and the current column coincides with the indentation
872 required := parser.flow_level == 0 && parser.indent == parser.mark.column
875 // If the current position may start a simple key, save it.
877 if parser.simple_key_allowed {
878 simple_key := yaml_simple_key_t{
881 token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head),
883 simple_key.mark = parser.mark
885 if !yaml_parser_remove_simple_key(parser) {
888 parser.simple_keys[len(parser.simple_keys)-1] = simple_key
893 // Remove a potential simple key at the current flow level.
894 func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool {
895 i := len(parser.simple_keys) - 1
896 if parser.simple_keys[i].possible {
897 // If the key is required, it is an error.
898 if parser.simple_keys[i].required {
899 return yaml_parser_set_scanner_error(parser,
900 "while scanning a simple key", parser.simple_keys[i].mark,
901 "could not find expected ':'")
904 // Remove the key from the stack.
905 parser.simple_keys[i].possible = false
909 // Increase the flow level and resize the simple key list if needed.
910 func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool {
911 // Reset the simple key on the next level.
912 parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{})
914 // Increase the flow level.
919 // Decrease the flow level.
920 func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool {
921 if parser.flow_level > 0 {
923 parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1]
928 // Push the current indentation level to the stack and set the new level
929 // the current column is greater than the indentation level. In this case,
930 // append or insert the specified token into the token queue.
931 func yaml_parser_roll_indent(parser *yaml_parser_t, column, number int, typ yaml_token_type_t, mark yaml_mark_t) bool {
932 // In the flow context, do nothing.
933 if parser.flow_level > 0 {
937 if parser.indent < column {
938 // Push the current indentation level to the stack and set the new
939 // indentation level.
940 parser.indents = append(parser.indents, parser.indent)
941 parser.indent = column
943 // Create a token and insert it into the queue.
944 token := yaml_token_t{
950 number -= parser.tokens_parsed
952 yaml_insert_token(parser, number, &token)
957 // Pop indentation levels from the indents stack until the current level
958 // becomes less or equal to the column. For each indentation level, append
959 // the BLOCK-END token.
960 func yaml_parser_unroll_indent(parser *yaml_parser_t, column int) bool {
961 // In the flow context, do nothing.
962 if parser.flow_level > 0 {
966 // Loop through the indentation levels in the stack.
967 for parser.indent > column {
968 // Create a token and append it to the queue.
969 token := yaml_token_t{
970 typ: yaml_BLOCK_END_TOKEN,
971 start_mark: parser.mark,
972 end_mark: parser.mark,
974 yaml_insert_token(parser, -1, &token)
976 // Pop the indentation level.
977 parser.indent = parser.indents[len(parser.indents)-1]
978 parser.indents = parser.indents[:len(parser.indents)-1]
983 // Initialize the scanner and produce the STREAM-START token.
984 func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool {
986 // Set the initial indentation.
989 // Initialize the simple key stack.
990 parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{})
992 // A simple key is allowed at the beginning of the stream.
993 parser.simple_key_allowed = true
996 parser.stream_start_produced = true
998 // Create the STREAM-START token and append it to the queue.
999 token := yaml_token_t{
1000 typ: yaml_STREAM_START_TOKEN,
1001 start_mark: parser.mark,
1002 end_mark: parser.mark,
1003 encoding: parser.encoding,
1005 yaml_insert_token(parser, -1, &token)
1009 // Produce the STREAM-END token and shut down the scanner.
1010 func yaml_parser_fetch_stream_end(parser *yaml_parser_t) bool {
1013 if parser.mark.column != 0 {
1014 parser.mark.column = 0
1018 // Reset the indentation level.
1019 if !yaml_parser_unroll_indent(parser, -1) {
1023 // Reset simple keys.
1024 if !yaml_parser_remove_simple_key(parser) {
1028 parser.simple_key_allowed = false
1030 // Create the STREAM-END token and append it to the queue.
1031 token := yaml_token_t{
1032 typ: yaml_STREAM_END_TOKEN,
1033 start_mark: parser.mark,
1034 end_mark: parser.mark,
1036 yaml_insert_token(parser, -1, &token)
1040 // Produce a VERSION-DIRECTIVE or TAG-DIRECTIVE token.
1041 func yaml_parser_fetch_directive(parser *yaml_parser_t) bool {
1042 // Reset the indentation level.
1043 if !yaml_parser_unroll_indent(parser, -1) {
1047 // Reset simple keys.
1048 if !yaml_parser_remove_simple_key(parser) {
1052 parser.simple_key_allowed = false
1054 // Create the YAML-DIRECTIVE or TAG-DIRECTIVE token.
1055 token := yaml_token_t{}
1056 if !yaml_parser_scan_directive(parser, &token) {
1059 // Append the token to the queue.
1060 yaml_insert_token(parser, -1, &token)
1064 // Produce the DOCUMENT-START or DOCUMENT-END token.
1065 func yaml_parser_fetch_document_indicator(parser *yaml_parser_t, typ yaml_token_type_t) bool {
1066 // Reset the indentation level.
1067 if !yaml_parser_unroll_indent(parser, -1) {
1071 // Reset simple keys.
1072 if !yaml_parser_remove_simple_key(parser) {
1076 parser.simple_key_allowed = false
1078 // Consume the token.
1079 start_mark := parser.mark
1085 end_mark := parser.mark
1087 // Create the DOCUMENT-START or DOCUMENT-END token.
1088 token := yaml_token_t{
1090 start_mark: start_mark,
1093 // Append the token to the queue.
1094 yaml_insert_token(parser, -1, &token)
1098 // Produce the FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
1099 func yaml_parser_fetch_flow_collection_start(parser *yaml_parser_t, typ yaml_token_type_t) bool {
1100 // The indicators '[' and '{' may start a simple key.
1101 if !yaml_parser_save_simple_key(parser) {
1105 // Increase the flow level.
1106 if !yaml_parser_increase_flow_level(parser) {
1110 // A simple key may follow the indicators '[' and '{'.
1111 parser.simple_key_allowed = true
1113 // Consume the token.
1114 start_mark := parser.mark
1116 end_mark := parser.mark
1118 // Create the FLOW-SEQUENCE-START of FLOW-MAPPING-START token.
1119 token := yaml_token_t{
1121 start_mark: start_mark,
1124 // Append the token to the queue.
1125 yaml_insert_token(parser, -1, &token)
1129 // Produce the FLOW-SEQUENCE-END or FLOW-MAPPING-END token.
1130 func yaml_parser_fetch_flow_collection_end(parser *yaml_parser_t, typ yaml_token_type_t) bool {
1131 // Reset any potential simple key on the current flow level.
1132 if !yaml_parser_remove_simple_key(parser) {
1136 // Decrease the flow level.
1137 if !yaml_parser_decrease_flow_level(parser) {
1141 // No simple keys after the indicators ']' and '}'.
1142 parser.simple_key_allowed = false
1144 // Consume the token.
1146 start_mark := parser.mark
1148 end_mark := parser.mark
1150 // Create the FLOW-SEQUENCE-END of FLOW-MAPPING-END token.
1151 token := yaml_token_t{
1153 start_mark: start_mark,
1156 // Append the token to the queue.
1157 yaml_insert_token(parser, -1, &token)
1161 // Produce the FLOW-ENTRY token.
1162 func yaml_parser_fetch_flow_entry(parser *yaml_parser_t) bool {
1163 // Reset any potential simple keys on the current flow level.
1164 if !yaml_parser_remove_simple_key(parser) {
1168 // Simple keys are allowed after ','.
1169 parser.simple_key_allowed = true
1171 // Consume the token.
1172 start_mark := parser.mark
1174 end_mark := parser.mark
1176 // Create the FLOW-ENTRY token and append it to the queue.
1177 token := yaml_token_t{
1178 typ: yaml_FLOW_ENTRY_TOKEN,
1179 start_mark: start_mark,
1182 yaml_insert_token(parser, -1, &token)
1186 // Produce the BLOCK-ENTRY token.
1187 func yaml_parser_fetch_block_entry(parser *yaml_parser_t) bool {
1188 // Check if the scanner is in the block context.
1189 if parser.flow_level == 0 {
1190 // Check if we are allowed to start a new entry.
1191 if !parser.simple_key_allowed {
1192 return yaml_parser_set_scanner_error(parser, "", parser.mark,
1193 "block sequence entries are not allowed in this context")
1195 // Add the BLOCK-SEQUENCE-START token if needed.
1196 if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_SEQUENCE_START_TOKEN, parser.mark) {
1200 // It is an error for the '-' indicator to occur in the flow context,
1201 // but we let the Parser detect and report about it because the Parser
1202 // is able to point to the context.
1205 // Reset any potential simple keys on the current flow level.
1206 if !yaml_parser_remove_simple_key(parser) {
1210 // Simple keys are allowed after '-'.
1211 parser.simple_key_allowed = true
1213 // Consume the token.
1214 start_mark := parser.mark
1216 end_mark := parser.mark
1218 // Create the BLOCK-ENTRY token and append it to the queue.
1219 token := yaml_token_t{
1220 typ: yaml_BLOCK_ENTRY_TOKEN,
1221 start_mark: start_mark,
1224 yaml_insert_token(parser, -1, &token)
1228 // Produce the KEY token.
1229 func yaml_parser_fetch_key(parser *yaml_parser_t) bool {
1231 // In the block context, additional checks are required.
1232 if parser.flow_level == 0 {
1233 // Check if we are allowed to start a new key (not nessesary simple).
1234 if !parser.simple_key_allowed {
1235 return yaml_parser_set_scanner_error(parser, "", parser.mark,
1236 "mapping keys are not allowed in this context")
1238 // Add the BLOCK-MAPPING-START token if needed.
1239 if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) {
1244 // Reset any potential simple keys on the current flow level.
1245 if !yaml_parser_remove_simple_key(parser) {
1249 // Simple keys are allowed after '?' in the block context.
1250 parser.simple_key_allowed = parser.flow_level == 0
1252 // Consume the token.
1253 start_mark := parser.mark
1255 end_mark := parser.mark
1257 // Create the KEY token and append it to the queue.
1258 token := yaml_token_t{
1259 typ: yaml_KEY_TOKEN,
1260 start_mark: start_mark,
1263 yaml_insert_token(parser, -1, &token)
1267 // Produce the VALUE token.
1268 func yaml_parser_fetch_value(parser *yaml_parser_t) bool {
1270 simple_key := &parser.simple_keys[len(parser.simple_keys)-1]
1272 // Have we found a simple key?
1273 if simple_key.possible {
1274 // Create the KEY token and insert it into the queue.
1275 token := yaml_token_t{
1276 typ: yaml_KEY_TOKEN,
1277 start_mark: simple_key.mark,
1278 end_mark: simple_key.mark,
1280 yaml_insert_token(parser, simple_key.token_number-parser.tokens_parsed, &token)
1282 // In the block context, we may need to add the BLOCK-MAPPING-START token.
1283 if !yaml_parser_roll_indent(parser, simple_key.mark.column,
1284 simple_key.token_number,
1285 yaml_BLOCK_MAPPING_START_TOKEN, simple_key.mark) {
1289 // Remove the simple key.
1290 simple_key.possible = false
1292 // A simple key cannot follow another simple key.
1293 parser.simple_key_allowed = false
1296 // The ':' indicator follows a complex key.
1298 // In the block context, extra checks are required.
1299 if parser.flow_level == 0 {
1301 // Check if we are allowed to start a complex value.
1302 if !parser.simple_key_allowed {
1303 return yaml_parser_set_scanner_error(parser, "", parser.mark,
1304 "mapping values are not allowed in this context")
1307 // Add the BLOCK-MAPPING-START token if needed.
1308 if !yaml_parser_roll_indent(parser, parser.mark.column, -1, yaml_BLOCK_MAPPING_START_TOKEN, parser.mark) {
1313 // Simple keys after ':' are allowed in the block context.
1314 parser.simple_key_allowed = parser.flow_level == 0
1317 // Consume the token.
1318 start_mark := parser.mark
1320 end_mark := parser.mark
1322 // Create the VALUE token and append it to the queue.
1323 token := yaml_token_t{
1324 typ: yaml_VALUE_TOKEN,
1325 start_mark: start_mark,
1328 yaml_insert_token(parser, -1, &token)
1332 // Produce the ALIAS or ANCHOR token.
1333 func yaml_parser_fetch_anchor(parser *yaml_parser_t, typ yaml_token_type_t) bool {
1334 // An anchor or an alias could be a simple key.
1335 if !yaml_parser_save_simple_key(parser) {
1339 // A simple key cannot follow an anchor or an alias.
1340 parser.simple_key_allowed = false
1342 // Create the ALIAS or ANCHOR token and append it to the queue.
1343 var token yaml_token_t
1344 if !yaml_parser_scan_anchor(parser, &token, typ) {
1347 yaml_insert_token(parser, -1, &token)
1351 // Produce the TAG token.
1352 func yaml_parser_fetch_tag(parser *yaml_parser_t) bool {
1353 // A tag could be a simple key.
1354 if !yaml_parser_save_simple_key(parser) {
1358 // A simple key cannot follow a tag.
1359 parser.simple_key_allowed = false
1361 // Create the TAG token and append it to the queue.
1362 var token yaml_token_t
1363 if !yaml_parser_scan_tag(parser, &token) {
1366 yaml_insert_token(parser, -1, &token)
1370 // Produce the SCALAR(...,literal) or SCALAR(...,folded) tokens.
1371 func yaml_parser_fetch_block_scalar(parser *yaml_parser_t, literal bool) bool {
1372 // Remove any potential simple keys.
1373 if !yaml_parser_remove_simple_key(parser) {
1377 // A simple key may follow a block scalar.
1378 parser.simple_key_allowed = true
1380 // Create the SCALAR token and append it to the queue.
1381 var token yaml_token_t
1382 if !yaml_parser_scan_block_scalar(parser, &token, literal) {
1385 yaml_insert_token(parser, -1, &token)
1389 // Produce the SCALAR(...,single-quoted) or SCALAR(...,double-quoted) tokens.
1390 func yaml_parser_fetch_flow_scalar(parser *yaml_parser_t, single bool) bool {
1391 // A plain scalar could be a simple key.
1392 if !yaml_parser_save_simple_key(parser) {
1396 // A simple key cannot follow a flow scalar.
1397 parser.simple_key_allowed = false
1399 // Create the SCALAR token and append it to the queue.
1400 var token yaml_token_t
1401 if !yaml_parser_scan_flow_scalar(parser, &token, single) {
1404 yaml_insert_token(parser, -1, &token)
1408 // Produce the SCALAR(...,plain) token.
1409 func yaml_parser_fetch_plain_scalar(parser *yaml_parser_t) bool {
1410 // A plain scalar could be a simple key.
1411 if !yaml_parser_save_simple_key(parser) {
1415 // A simple key cannot follow a flow scalar.
1416 parser.simple_key_allowed = false
1418 // Create the SCALAR token and append it to the queue.
1419 var token yaml_token_t
1420 if !yaml_parser_scan_plain_scalar(parser, &token) {
1423 yaml_insert_token(parser, -1, &token)
1427 // Eat whitespaces and comments until the next token is found.
1428 func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool {
1430 // Until the next token is not found.
1432 // Allow the BOM mark to start a line.
1433 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1436 if parser.mark.column == 0 && is_bom(parser.buffer, parser.buffer_pos) {
1441 // Tabs are allowed:
1442 // - in the flow context
1443 // - in the block context, but not at the beginning of the line or
1444 // after '-', '?', or ':' (complex value).
1445 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1449 for parser.buffer[parser.buffer_pos] == ' ' || ((parser.flow_level > 0 || !parser.simple_key_allowed) && parser.buffer[parser.buffer_pos] == '\t') {
1451 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1456 // Eat a comment until a line break.
1457 if parser.buffer[parser.buffer_pos] == '#' {
1458 for !is_breakz(parser.buffer, parser.buffer_pos) {
1460 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1466 // If it is a line break, eat it.
1467 if is_break(parser.buffer, parser.buffer_pos) {
1468 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
1473 // In the block context, a new line may start a simple key.
1474 if parser.flow_level == 0 {
1475 parser.simple_key_allowed = true
1478 break // We have found a token.
1485 // Scan a YAML-DIRECTIVE or TAG-DIRECTIVE token.
1488 // %YAML 1.1 # a comment \n
1489 // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1490 // %TAG !yaml! tag:yaml.org,2002: \n
1491 // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1493 func yaml_parser_scan_directive(parser *yaml_parser_t, token *yaml_token_t) bool {
1495 start_mark := parser.mark
1498 // Scan the directive name.
1500 if !yaml_parser_scan_directive_name(parser, start_mark, &name) {
1504 // Is it a YAML directive?
1505 if bytes.Equal(name, []byte("YAML")) {
1506 // Scan the VERSION directive value.
1507 var major, minor int8
1508 if !yaml_parser_scan_version_directive_value(parser, start_mark, &major, &minor) {
1511 end_mark := parser.mark
1513 // Create a VERSION-DIRECTIVE token.
1514 *token = yaml_token_t{
1515 typ: yaml_VERSION_DIRECTIVE_TOKEN,
1516 start_mark: start_mark,
1522 // Is it a TAG directive?
1523 } else if bytes.Equal(name, []byte("TAG")) {
1524 // Scan the TAG directive value.
1525 var handle, prefix []byte
1526 if !yaml_parser_scan_tag_directive_value(parser, start_mark, &handle, &prefix) {
1529 end_mark := parser.mark
1531 // Create a TAG-DIRECTIVE token.
1532 *token = yaml_token_t{
1533 typ: yaml_TAG_DIRECTIVE_TOKEN,
1534 start_mark: start_mark,
1540 // Unknown directive.
1542 yaml_parser_set_scanner_error(parser, "while scanning a directive",
1543 start_mark, "found unknown directive name")
1547 // Eat the rest of the line including any comments.
1548 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1552 for is_blank(parser.buffer, parser.buffer_pos) {
1554 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1559 if parser.buffer[parser.buffer_pos] == '#' {
1560 for !is_breakz(parser.buffer, parser.buffer_pos) {
1562 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1568 // Check if we are at the end of the line.
1569 if !is_breakz(parser.buffer, parser.buffer_pos) {
1570 yaml_parser_set_scanner_error(parser, "while scanning a directive",
1571 start_mark, "did not find expected comment or line break")
1575 // Eat a line break.
1576 if is_break(parser.buffer, parser.buffer_pos) {
1577 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
1586 // Scan the directive name.
1589 // %YAML 1.1 # a comment \n
1591 // %TAG !yaml! tag:yaml.org,2002: \n
1594 func yaml_parser_scan_directive_name(parser *yaml_parser_t, start_mark yaml_mark_t, name *[]byte) bool {
1595 // Consume the directive name.
1596 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1601 for is_alpha(parser.buffer, parser.buffer_pos) {
1603 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1608 // Check if the name is empty.
1610 yaml_parser_set_scanner_error(parser, "while scanning a directive",
1611 start_mark, "could not find expected directive name")
1615 // Check for an blank character after the name.
1616 if !is_blankz(parser.buffer, parser.buffer_pos) {
1617 yaml_parser_set_scanner_error(parser, "while scanning a directive",
1618 start_mark, "found unexpected non-alphabetical character")
1625 // Scan the value of VERSION-DIRECTIVE.
1628 // %YAML 1.1 # a comment \n
1630 func yaml_parser_scan_version_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, major, minor *int8) bool {
1632 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1635 for is_blank(parser.buffer, parser.buffer_pos) {
1637 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1642 // Consume the major version number.
1643 if !yaml_parser_scan_version_directive_number(parser, start_mark, major) {
1648 if parser.buffer[parser.buffer_pos] != '.' {
1649 return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
1650 start_mark, "did not find expected digit or '.' character")
1655 // Consume the minor version number.
1656 if !yaml_parser_scan_version_directive_number(parser, start_mark, minor) {
1662 const max_number_length = 2
1664 // Scan the version number of VERSION-DIRECTIVE.
1667 // %YAML 1.1 # a comment \n
1669 // %YAML 1.1 # a comment \n
1671 func yaml_parser_scan_version_directive_number(parser *yaml_parser_t, start_mark yaml_mark_t, number *int8) bool {
1673 // Repeat while the next character is digit.
1674 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1677 var value, length int8
1678 for is_digit(parser.buffer, parser.buffer_pos) {
1679 // Check if the number is too long.
1681 if length > max_number_length {
1682 return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
1683 start_mark, "found extremely long version number")
1685 value = value*10 + int8(as_digit(parser.buffer, parser.buffer_pos))
1687 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1692 // Check if the number was present.
1694 return yaml_parser_set_scanner_error(parser, "while scanning a %YAML directive",
1695 start_mark, "did not find expected version number")
1701 // Scan the value of a TAG-DIRECTIVE token.
1704 // %TAG !yaml! tag:yaml.org,2002: \n
1705 // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1707 func yaml_parser_scan_tag_directive_value(parser *yaml_parser_t, start_mark yaml_mark_t, handle, prefix *[]byte) bool {
1708 var handle_value, prefix_value []byte
1711 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1715 for is_blank(parser.buffer, parser.buffer_pos) {
1717 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1723 if !yaml_parser_scan_tag_handle(parser, true, start_mark, &handle_value) {
1727 // Expect a whitespace.
1728 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1731 if !is_blank(parser.buffer, parser.buffer_pos) {
1732 yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive",
1733 start_mark, "did not find expected whitespace")
1738 for is_blank(parser.buffer, parser.buffer_pos) {
1740 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1746 if !yaml_parser_scan_tag_uri(parser, true, nil, start_mark, &prefix_value) {
1750 // Expect a whitespace or line break.
1751 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1754 if !is_blankz(parser.buffer, parser.buffer_pos) {
1755 yaml_parser_set_scanner_error(parser, "while scanning a %TAG directive",
1756 start_mark, "did not find expected whitespace or line break")
1760 *handle = handle_value
1761 *prefix = prefix_value
1765 func yaml_parser_scan_anchor(parser *yaml_parser_t, token *yaml_token_t, typ yaml_token_type_t) bool {
1768 // Eat the indicator character.
1769 start_mark := parser.mark
1772 // Consume the value.
1773 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1777 for is_alpha(parser.buffer, parser.buffer_pos) {
1779 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1784 end_mark := parser.mark
1787 * Check if length of the anchor is greater than 0 and it is followed by
1788 * a whitespace character or one of the indicators:
1790 * '?', ':', ',', ']', '}', '%', '@', '`'.
1794 !(is_blankz(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == '?' ||
1795 parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == ',' ||
1796 parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '}' ||
1797 parser.buffer[parser.buffer_pos] == '%' || parser.buffer[parser.buffer_pos] == '@' ||
1798 parser.buffer[parser.buffer_pos] == '`') {
1799 context := "while scanning an alias"
1800 if typ == yaml_ANCHOR_TOKEN {
1801 context = "while scanning an anchor"
1803 yaml_parser_set_scanner_error(parser, context, start_mark,
1804 "did not find expected alphabetic or numeric character")
1809 *token = yaml_token_t{
1811 start_mark: start_mark,
1823 func yaml_parser_scan_tag(parser *yaml_parser_t, token *yaml_token_t) bool {
1824 var handle, suffix []byte
1826 start_mark := parser.mark
1828 // Check if the tag is in the canonical form.
1829 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
1833 if parser.buffer[parser.buffer_pos+1] == '<' {
1834 // Keep the handle as ''
1840 // Consume the tag value.
1841 if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) {
1845 // Check for '>' and eat it.
1846 if parser.buffer[parser.buffer_pos] != '>' {
1847 yaml_parser_set_scanner_error(parser, "while scanning a tag",
1848 start_mark, "did not find the expected '>'")
1854 // The tag has either the '!suffix' or the '!handle!suffix' form.
1856 // First, try to scan a handle.
1857 if !yaml_parser_scan_tag_handle(parser, false, start_mark, &handle) {
1861 // Check if it is, indeed, handle.
1862 if handle[0] == '!' && len(handle) > 1 && handle[len(handle)-1] == '!' {
1863 // Scan the suffix now.
1864 if !yaml_parser_scan_tag_uri(parser, false, nil, start_mark, &suffix) {
1868 // It wasn't a handle after all. Scan the rest of the tag.
1869 if !yaml_parser_scan_tag_uri(parser, false, handle, start_mark, &suffix) {
1873 // Set the handle to '!'.
1874 handle = []byte{'!'}
1876 // A special case: the '!' tag. Set the handle to '' and the
1878 if len(suffix) == 0 {
1879 handle, suffix = suffix, handle
1884 // Check the character which ends the tag.
1885 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1888 if !is_blankz(parser.buffer, parser.buffer_pos) {
1889 yaml_parser_set_scanner_error(parser, "while scanning a tag",
1890 start_mark, "did not find expected whitespace or line break")
1894 end_mark := parser.mark
1897 *token = yaml_token_t{
1898 typ: yaml_TAG_TOKEN,
1899 start_mark: start_mark,
1907 // Scan a tag handle.
1908 func yaml_parser_scan_tag_handle(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, handle *[]byte) bool {
1909 // Check the initial '!' character.
1910 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1913 if parser.buffer[parser.buffer_pos] != '!' {
1914 yaml_parser_set_scanner_tag_error(parser, directive,
1915 start_mark, "did not find expected '!'")
1921 // Copy the '!' character.
1924 // Copy all subsequent alphabetical and numerical characters.
1925 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1928 for is_alpha(parser.buffer, parser.buffer_pos) {
1930 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1935 // Check if the trailing character is '!' and copy it.
1936 if parser.buffer[parser.buffer_pos] == '!' {
1939 // It's either the '!' tag or not really a tag handle. If it's a %TAG
1940 // directive, it's an error. If it's a tag token, it must be a part of URI.
1941 if directive && string(s) != "!" {
1942 yaml_parser_set_scanner_tag_error(parser, directive,
1943 start_mark, "did not find expected '!'")
1953 func yaml_parser_scan_tag_uri(parser *yaml_parser_t, directive bool, head []byte, start_mark yaml_mark_t, uri *[]byte) bool {
1954 //size_t length = head ? strlen((char *)head) : 0
1956 hasTag := len(head) > 0
1958 // Copy the head if needed.
1960 // Note that we don't copy the leading '!' character.
1962 s = append(s, head[1:]...)
1966 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
1970 // The set of characters that may appear in URI is as follows:
1972 // '0'-'9', 'A'-'Z', 'a'-'z', '_', '-', ';', '/', '?', ':', '@', '&',
1973 // '=', '+', '$', ',', '.', '!', '~', '*', '\'', '(', ')', '[', ']',
1975 // [Go] Convert this into more reasonable logic.
1976 for is_alpha(parser.buffer, parser.buffer_pos) || parser.buffer[parser.buffer_pos] == ';' ||
1977 parser.buffer[parser.buffer_pos] == '/' || parser.buffer[parser.buffer_pos] == '?' ||
1978 parser.buffer[parser.buffer_pos] == ':' || parser.buffer[parser.buffer_pos] == '@' ||
1979 parser.buffer[parser.buffer_pos] == '&' || parser.buffer[parser.buffer_pos] == '=' ||
1980 parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '$' ||
1981 parser.buffer[parser.buffer_pos] == ',' || parser.buffer[parser.buffer_pos] == '.' ||
1982 parser.buffer[parser.buffer_pos] == '!' || parser.buffer[parser.buffer_pos] == '~' ||
1983 parser.buffer[parser.buffer_pos] == '*' || parser.buffer[parser.buffer_pos] == '\'' ||
1984 parser.buffer[parser.buffer_pos] == '(' || parser.buffer[parser.buffer_pos] == ')' ||
1985 parser.buffer[parser.buffer_pos] == '[' || parser.buffer[parser.buffer_pos] == ']' ||
1986 parser.buffer[parser.buffer_pos] == '%' {
1987 // Check if it is a URI-escape sequence.
1988 if parser.buffer[parser.buffer_pos] == '%' {
1989 if !yaml_parser_scan_uri_escapes(parser, directive, start_mark, &s) {
1995 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2002 yaml_parser_set_scanner_tag_error(parser, directive,
2003 start_mark, "did not find expected tag URI")
2010 // Decode an URI-escape sequence corresponding to a single UTF-8 character.
2011 func yaml_parser_scan_uri_escapes(parser *yaml_parser_t, directive bool, start_mark yaml_mark_t, s *[]byte) bool {
2013 // Decode the required number of characters.
2016 // Check for a URI-escaped octet.
2017 if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) {
2021 if !(parser.buffer[parser.buffer_pos] == '%' &&
2022 is_hex(parser.buffer, parser.buffer_pos+1) &&
2023 is_hex(parser.buffer, parser.buffer_pos+2)) {
2024 return yaml_parser_set_scanner_tag_error(parser, directive,
2025 start_mark, "did not find URI escaped octet")
2029 octet := byte((as_hex(parser.buffer, parser.buffer_pos+1) << 4) + as_hex(parser.buffer, parser.buffer_pos+2))
2031 // If it is the leading octet, determine the length of the UTF-8 sequence.
2035 return yaml_parser_set_scanner_tag_error(parser, directive,
2036 start_mark, "found an incorrect leading UTF-8 octet")
2039 // Check if the trailing octet is correct.
2040 if octet&0xC0 != 0x80 {
2041 return yaml_parser_set_scanner_tag_error(parser, directive,
2042 start_mark, "found an incorrect trailing UTF-8 octet")
2046 // Copy the octet and move the pointers.
2047 *s = append(*s, octet)
2056 // Scan a block scalar.
2057 func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, literal bool) bool {
2058 // Eat the indicator '|' or '>'.
2059 start_mark := parser.mark
2062 // Scan the additional block scalar indicators.
2063 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2067 // Check for a chomping indicator.
2068 var chomping, increment int
2069 if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' {
2070 // Set the chomping method and eat the indicator.
2071 if parser.buffer[parser.buffer_pos] == '+' {
2078 // Check for an indentation indicator.
2079 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2082 if is_digit(parser.buffer, parser.buffer_pos) {
2083 // Check that the indentation is greater than 0.
2084 if parser.buffer[parser.buffer_pos] == '0' {
2085 yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2086 start_mark, "found an indentation indicator equal to 0")
2090 // Get the indentation level and eat the indicator.
2091 increment = as_digit(parser.buffer, parser.buffer_pos)
2095 } else if is_digit(parser.buffer, parser.buffer_pos) {
2096 // Do the same as above, but in the opposite order.
2098 if parser.buffer[parser.buffer_pos] == '0' {
2099 yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2100 start_mark, "found an indentation indicator equal to 0")
2103 increment = as_digit(parser.buffer, parser.buffer_pos)
2106 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2109 if parser.buffer[parser.buffer_pos] == '+' || parser.buffer[parser.buffer_pos] == '-' {
2110 if parser.buffer[parser.buffer_pos] == '+' {
2119 // Eat whitespaces and comments to the end of the line.
2120 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2123 for is_blank(parser.buffer, parser.buffer_pos) {
2125 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2129 if parser.buffer[parser.buffer_pos] == '#' {
2130 for !is_breakz(parser.buffer, parser.buffer_pos) {
2132 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2138 // Check if we are at the end of the line.
2139 if !is_breakz(parser.buffer, parser.buffer_pos) {
2140 yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2141 start_mark, "did not find expected comment or line break")
2145 // Eat a line break.
2146 if is_break(parser.buffer, parser.buffer_pos) {
2147 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2153 end_mark := parser.mark
2155 // Set the indentation level if it was specified.
2158 if parser.indent >= 0 {
2159 indent = parser.indent + increment
2165 // Scan the leading line breaks and determine the indentation level if needed.
2166 var s, leading_break, trailing_breaks []byte
2167 if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) {
2171 // Scan the block scalar content.
2172 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2175 var leading_blank, trailing_blank bool
2176 for parser.mark.column == indent && !is_z(parser.buffer, parser.buffer_pos) {
2177 // We are at the beginning of a non-empty line.
2179 // Is it a trailing whitespace?
2180 trailing_blank = is_blank(parser.buffer, parser.buffer_pos)
2182 // Check if we need to fold the leading line break.
2183 if !literal && !leading_blank && !trailing_blank && len(leading_break) > 0 && leading_break[0] == '\n' {
2184 // Do we need to join the lines by space?
2185 if len(trailing_breaks) == 0 {
2189 s = append(s, leading_break...)
2191 leading_break = leading_break[:0]
2193 // Append the remaining line breaks.
2194 s = append(s, trailing_breaks...)
2195 trailing_breaks = trailing_breaks[:0]
2197 // Is it a leading whitespace?
2198 leading_blank = is_blank(parser.buffer, parser.buffer_pos)
2200 // Consume the current line.
2201 for !is_breakz(parser.buffer, parser.buffer_pos) {
2203 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2208 // Consume the line break.
2209 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2213 leading_break = read_line(parser, leading_break)
2215 // Eat the following indentation spaces and line breaks.
2216 if !yaml_parser_scan_block_scalar_breaks(parser, &indent, &trailing_breaks, start_mark, &end_mark) {
2223 s = append(s, leading_break...)
2226 s = append(s, trailing_breaks...)
2230 *token = yaml_token_t{
2231 typ: yaml_SCALAR_TOKEN,
2232 start_mark: start_mark,
2235 style: yaml_LITERAL_SCALAR_STYLE,
2238 token.style = yaml_FOLDED_SCALAR_STYLE
2243 // Scan indentation spaces and line breaks for a block scalar. Determine the
2244 // indentation level if needed.
2245 func yaml_parser_scan_block_scalar_breaks(parser *yaml_parser_t, indent *int, breaks *[]byte, start_mark yaml_mark_t, end_mark *yaml_mark_t) bool {
2246 *end_mark = parser.mark
2248 // Eat the indentation spaces and line breaks.
2251 // Eat the indentation spaces.
2252 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2255 for (*indent == 0 || parser.mark.column < *indent) && is_space(parser.buffer, parser.buffer_pos) {
2257 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2261 if parser.mark.column > max_indent {
2262 max_indent = parser.mark.column
2265 // Check for a tab character messing the indentation.
2266 if (*indent == 0 || parser.mark.column < *indent) && is_tab(parser.buffer, parser.buffer_pos) {
2267 return yaml_parser_set_scanner_error(parser, "while scanning a block scalar",
2268 start_mark, "found a tab character where an indentation space is expected")
2271 // Have we found a non-empty line?
2272 if !is_break(parser.buffer, parser.buffer_pos) {
2276 // Consume the line break.
2277 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2280 // [Go] Should really be returning breaks instead.
2281 *breaks = read_line(parser, *breaks)
2282 *end_mark = parser.mark
2285 // Determine the indentation level if needed.
2287 *indent = max_indent
2288 if *indent < parser.indent+1 {
2289 *indent = parser.indent + 1
2298 // Scan a quoted scalar.
2299 func yaml_parser_scan_flow_scalar(parser *yaml_parser_t, token *yaml_token_t, single bool) bool {
2300 // Eat the left quote.
2301 start_mark := parser.mark
2304 // Consume the content of the quoted scalar.
2305 var s, leading_break, trailing_breaks, whitespaces []byte
2307 // Check that there are no document indicators at the beginning of the line.
2308 if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) {
2312 if parser.mark.column == 0 &&
2313 ((parser.buffer[parser.buffer_pos+0] == '-' &&
2314 parser.buffer[parser.buffer_pos+1] == '-' &&
2315 parser.buffer[parser.buffer_pos+2] == '-') ||
2316 (parser.buffer[parser.buffer_pos+0] == '.' &&
2317 parser.buffer[parser.buffer_pos+1] == '.' &&
2318 parser.buffer[parser.buffer_pos+2] == '.')) &&
2319 is_blankz(parser.buffer, parser.buffer_pos+3) {
2320 yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar",
2321 start_mark, "found unexpected document indicator")
2326 if is_z(parser.buffer, parser.buffer_pos) {
2327 yaml_parser_set_scanner_error(parser, "while scanning a quoted scalar",
2328 start_mark, "found unexpected end of stream")
2332 // Consume non-blank characters.
2333 leading_blanks := false
2334 for !is_blankz(parser.buffer, parser.buffer_pos) {
2335 if single && parser.buffer[parser.buffer_pos] == '\'' && parser.buffer[parser.buffer_pos+1] == '\'' {
2336 // Is is an escaped single quote.
2341 } else if single && parser.buffer[parser.buffer_pos] == '\'' {
2342 // It is a right single quote.
2344 } else if !single && parser.buffer[parser.buffer_pos] == '"' {
2345 // It is a right double quote.
2348 } else if !single && parser.buffer[parser.buffer_pos] == '\\' && is_break(parser.buffer, parser.buffer_pos+1) {
2349 // It is an escaped line break.
2350 if parser.unread < 3 && !yaml_parser_update_buffer(parser, 3) {
2355 leading_blanks = true
2358 } else if !single && parser.buffer[parser.buffer_pos] == '\\' {
2359 // It is an escape sequence.
2362 // Check the escape character.
2363 switch parser.buffer[parser.buffer_pos+1] {
2367 s = append(s, '\x07')
2369 s = append(s, '\x08')
2371 s = append(s, '\x09')
2373 s = append(s, '\x0A')
2375 s = append(s, '\x0B')
2377 s = append(s, '\x0C')
2379 s = append(s, '\x0D')
2381 s = append(s, '\x1B')
2383 s = append(s, '\x20')
2390 case 'N': // NEL (#x85)
2391 s = append(s, '\xC2')
2392 s = append(s, '\x85')
2394 s = append(s, '\xC2')
2395 s = append(s, '\xA0')
2396 case 'L': // LS (#x2028)
2397 s = append(s, '\xE2')
2398 s = append(s, '\x80')
2399 s = append(s, '\xA8')
2400 case 'P': // PS (#x2029)
2401 s = append(s, '\xE2')
2402 s = append(s, '\x80')
2403 s = append(s, '\xA9')
2411 yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
2412 start_mark, "found unknown escape character")
2419 // Consume an arbitrary escape code.
2420 if code_length > 0 {
2423 // Scan the character value.
2424 if parser.unread < code_length && !yaml_parser_update_buffer(parser, code_length) {
2427 for k := 0; k < code_length; k++ {
2428 if !is_hex(parser.buffer, parser.buffer_pos+k) {
2429 yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
2430 start_mark, "did not find expected hexdecimal number")
2433 value = (value << 4) + as_hex(parser.buffer, parser.buffer_pos+k)
2436 // Check the value and write the character.
2437 if (value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF {
2438 yaml_parser_set_scanner_error(parser, "while parsing a quoted scalar",
2439 start_mark, "found invalid Unicode character escape code")
2443 s = append(s, byte(value))
2444 } else if value <= 0x7FF {
2445 s = append(s, byte(0xC0+(value>>6)))
2446 s = append(s, byte(0x80+(value&0x3F)))
2447 } else if value <= 0xFFFF {
2448 s = append(s, byte(0xE0+(value>>12)))
2449 s = append(s, byte(0x80+((value>>6)&0x3F)))
2450 s = append(s, byte(0x80+(value&0x3F)))
2452 s = append(s, byte(0xF0+(value>>18)))
2453 s = append(s, byte(0x80+((value>>12)&0x3F)))
2454 s = append(s, byte(0x80+((value>>6)&0x3F)))
2455 s = append(s, byte(0x80+(value&0x3F)))
2458 // Advance the pointer.
2459 for k := 0; k < code_length; k++ {
2464 // It is a non-escaped non-blank character.
2467 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2472 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2476 // Check if we are at the end of the scalar.
2478 if parser.buffer[parser.buffer_pos] == '\'' {
2482 if parser.buffer[parser.buffer_pos] == '"' {
2487 // Consume blank characters.
2488 for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) {
2489 if is_blank(parser.buffer, parser.buffer_pos) {
2490 // Consume a space or a tab character.
2491 if !leading_blanks {
2492 whitespaces = read(parser, whitespaces)
2497 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2501 // Check if it is a first line break.
2502 if !leading_blanks {
2503 whitespaces = whitespaces[:0]
2504 leading_break = read_line(parser, leading_break)
2505 leading_blanks = true
2507 trailing_breaks = read_line(parser, trailing_breaks)
2510 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2515 // Join the whitespaces or fold line breaks.
2517 // Do we need to fold line breaks?
2518 if len(leading_break) > 0 && leading_break[0] == '\n' {
2519 if len(trailing_breaks) == 0 {
2522 s = append(s, trailing_breaks...)
2525 s = append(s, leading_break...)
2526 s = append(s, trailing_breaks...)
2528 trailing_breaks = trailing_breaks[:0]
2529 leading_break = leading_break[:0]
2531 s = append(s, whitespaces...)
2532 whitespaces = whitespaces[:0]
2536 // Eat the right quote.
2538 end_mark := parser.mark
2541 *token = yaml_token_t{
2542 typ: yaml_SCALAR_TOKEN,
2543 start_mark: start_mark,
2546 style: yaml_SINGLE_QUOTED_SCALAR_STYLE,
2549 token.style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
2554 // Scan a plain scalar.
2555 func yaml_parser_scan_plain_scalar(parser *yaml_parser_t, token *yaml_token_t) bool {
2557 var s, leading_break, trailing_breaks, whitespaces []byte
2558 var leading_blanks bool
2559 var indent = parser.indent + 1
2561 start_mark := parser.mark
2562 end_mark := parser.mark
2564 // Consume the content of the plain scalar.
2566 // Check for a document indicator.
2567 if parser.unread < 4 && !yaml_parser_update_buffer(parser, 4) {
2570 if parser.mark.column == 0 &&
2571 ((parser.buffer[parser.buffer_pos+0] == '-' &&
2572 parser.buffer[parser.buffer_pos+1] == '-' &&
2573 parser.buffer[parser.buffer_pos+2] == '-') ||
2574 (parser.buffer[parser.buffer_pos+0] == '.' &&
2575 parser.buffer[parser.buffer_pos+1] == '.' &&
2576 parser.buffer[parser.buffer_pos+2] == '.')) &&
2577 is_blankz(parser.buffer, parser.buffer_pos+3) {
2581 // Check for a comment.
2582 if parser.buffer[parser.buffer_pos] == '#' {
2586 // Consume non-blank characters.
2587 for !is_blankz(parser.buffer, parser.buffer_pos) {
2589 // Check for indicators that may end a plain scalar.
2590 if (parser.buffer[parser.buffer_pos] == ':' && is_blankz(parser.buffer, parser.buffer_pos+1)) ||
2591 (parser.flow_level > 0 &&
2592 (parser.buffer[parser.buffer_pos] == ',' ||
2593 parser.buffer[parser.buffer_pos] == '?' || parser.buffer[parser.buffer_pos] == '[' ||
2594 parser.buffer[parser.buffer_pos] == ']' || parser.buffer[parser.buffer_pos] == '{' ||
2595 parser.buffer[parser.buffer_pos] == '}')) {
2599 // Check if we need to join whitespaces and breaks.
2600 if leading_blanks || len(whitespaces) > 0 {
2602 // Do we need to fold line breaks?
2603 if leading_break[0] == '\n' {
2604 if len(trailing_breaks) == 0 {
2607 s = append(s, trailing_breaks...)
2610 s = append(s, leading_break...)
2611 s = append(s, trailing_breaks...)
2613 trailing_breaks = trailing_breaks[:0]
2614 leading_break = leading_break[:0]
2615 leading_blanks = false
2617 s = append(s, whitespaces...)
2618 whitespaces = whitespaces[:0]
2622 // Copy the character.
2625 end_mark = parser.mark
2626 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2632 if !(is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos)) {
2636 // Consume blank characters.
2637 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2641 for is_blank(parser.buffer, parser.buffer_pos) || is_break(parser.buffer, parser.buffer_pos) {
2642 if is_blank(parser.buffer, parser.buffer_pos) {
2644 // Check for tab characters that abuse indentation.
2645 if leading_blanks && parser.mark.column < indent && is_tab(parser.buffer, parser.buffer_pos) {
2646 yaml_parser_set_scanner_error(parser, "while scanning a plain scalar",
2647 start_mark, "found a tab character that violates indentation")
2651 // Consume a space or a tab character.
2652 if !leading_blanks {
2653 whitespaces = read(parser, whitespaces)
2658 if parser.unread < 2 && !yaml_parser_update_buffer(parser, 2) {
2662 // Check if it is a first line break.
2663 if !leading_blanks {
2664 whitespaces = whitespaces[:0]
2665 leading_break = read_line(parser, leading_break)
2666 leading_blanks = true
2668 trailing_breaks = read_line(parser, trailing_breaks)
2671 if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) {
2676 // Check indentation level.
2677 if parser.flow_level == 0 && parser.mark.column < indent {
2683 *token = yaml_token_t{
2684 typ: yaml_SCALAR_TOKEN,
2685 start_mark: start_mark,
2688 style: yaml_PLAIN_SCALAR_STYLE,
2691 // Note that we change the 'simple_key_allowed' flag.
2693 parser.simple_key_allowed = true