2 Copyright 2014 The Kubernetes Authors.
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
8 http://www.apache.org/licenses/LICENSE-2.0
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
33 // ToJSON converts a single YAML document into a JSON document
34 // or returns an error. If the document appears to be JSON the
35 // YAML decoding path is not used (so that error messages are
37 func ToJSON(data []byte) ([]byte, error) {
38 if hasJSONPrefix(data) {
41 return yaml.YAMLToJSON(data)
44 // YAMLToJSONDecoder decodes YAML documents from an io.Reader by
45 // separating individual documents. It first converts the YAML
46 // body to JSON, then unmarshals the JSON.
47 type YAMLToJSONDecoder struct {
51 // NewYAMLToJSONDecoder decodes YAML documents from the provided
52 // stream in chunks by converting each document (as defined by
53 // the YAML spec) into its own chunk, converting it to JSON via
54 // yaml.YAMLToJSON, and then passing it to json.Decoder.
55 func NewYAMLToJSONDecoder(r io.Reader) *YAMLToJSONDecoder {
56 reader := bufio.NewReader(r)
57 return &YAMLToJSONDecoder{
58 reader: NewYAMLReader(reader),
62 // Decode reads a YAML document as JSON from the stream or returns
63 // an error. The decoding rules match json.Unmarshal, not
65 func (d *YAMLToJSONDecoder) Decode(into interface{}) error {
66 bytes, err := d.reader.Read()
67 if err != nil && err != io.EOF {
72 err := yaml.Unmarshal(bytes, into)
74 return YAMLSyntaxError{err}
80 // YAMLDecoder reads chunks of objects and returns ErrShortBuffer if
81 // the data is not sufficient.
82 type YAMLDecoder struct {
84 scanner *bufio.Scanner
88 // NewDocumentDecoder decodes YAML documents from the provided
89 // stream in chunks by converting each document (as defined by
90 // the YAML spec) into its own chunk. io.ErrShortBuffer will be
91 // returned if the entire buffer could not be read to assist
92 // the caller in framing the chunk.
93 func NewDocumentDecoder(r io.ReadCloser) io.ReadCloser {
94 scanner := bufio.NewScanner(r)
95 scanner.Split(splitYAMLDocument)
102 // Read reads the previous slice into the buffer, or attempts to read
104 // TODO: switch to readline approach.
105 func (d *YAMLDecoder) Read(data []byte) (n int, err error) {
106 left := len(d.remaining)
108 // return the next chunk from the stream
109 if !d.scanner.Scan() {
110 err := d.scanner.Err()
116 out := d.scanner.Bytes()
122 if left <= len(data) {
123 copy(data, d.remaining)
128 // caller will need to reread
129 copy(data, d.remaining[:len(data)])
130 d.remaining = d.remaining[len(data):]
131 return len(data), io.ErrShortBuffer
134 func (d *YAMLDecoder) Close() error {
138 const yamlSeparator = "\n---"
139 const separator = "---"
141 // splitYAMLDocument is a bufio.SplitFunc for splitting YAML streams into individual documents.
142 func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
143 if atEOF && len(data) == 0 {
146 sep := len([]byte(yamlSeparator))
147 if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
148 // We have a potential document terminator
152 // we can't read any more characters
154 return len(data), data[:len(data)-sep], nil
158 if j := bytes.IndexByte(after, '\n'); j >= 0 {
159 return i + j + 1, data[0 : i-sep], nil
163 // If we're at EOF, we have a final, non-terminated line. Return it.
165 return len(data), data, nil
167 // Request more data.
171 // decoder is a convenience interface for Decode.
172 type decoder interface {
173 Decode(into interface{}) error
176 // YAMLOrJSONDecoder attempts to decode a stream of JSON documents or
177 // YAML documents by sniffing for a leading { character.
178 type YAMLOrJSONDecoder struct {
186 type JSONSyntaxError struct {
191 func (e JSONSyntaxError) Error() string {
192 return fmt.Sprintf("json: line %d: %s", e.Line, e.Err.Error())
195 type YAMLSyntaxError struct {
199 func (e YAMLSyntaxError) Error() string {
203 // NewYAMLOrJSONDecoder returns a decoder that will process YAML documents
204 // or JSON documents from the given reader as a stream. bufferSize determines
205 // how far into the stream the decoder will look to figure out whether this
206 // is a JSON stream (has whitespace followed by an open brace).
207 func NewYAMLOrJSONDecoder(r io.Reader, bufferSize int) *YAMLOrJSONDecoder {
208 return &YAMLOrJSONDecoder{
210 bufferSize: bufferSize,
214 // Decode unmarshals the next object from the underlying stream into the
215 // provide object, or returns an error.
216 func (d *YAMLOrJSONDecoder) Decode(into interface{}) error {
217 if d.decoder == nil {
218 buffer, origData, isJSON := GuessJSONStream(d.r, d.bufferSize)
220 klog.V(4).Infof("decoding stream as JSON")
221 d.decoder = json.NewDecoder(buffer)
224 klog.V(4).Infof("decoding stream as YAML")
225 d.decoder = NewYAMLToJSONDecoder(buffer)
228 err := d.decoder.Decode(into)
229 if jsonDecoder, ok := d.decoder.(*json.Decoder); ok {
230 if syntax, ok := err.(*json.SyntaxError); ok {
231 data, readErr := ioutil.ReadAll(jsonDecoder.Buffered())
233 klog.V(4).Infof("reading stream failed: %v", readErr)
237 // if contents from io.Reader are not complete,
238 // use the original raw data to prevent panic
239 if int64(len(js)) <= syntax.Offset {
240 js = string(d.rawData)
243 start := strings.LastIndex(js[:syntax.Offset], "\n") + 1
244 line := strings.Count(js[:start], "\n")
245 return JSONSyntaxError{
247 Err: fmt.Errorf(syntax.Error()),
254 type Reader interface {
255 Read() ([]byte, error)
258 type YAMLReader struct {
262 func NewYAMLReader(r *bufio.Reader) *YAMLReader {
264 reader: &LineReader{reader: r},
268 // Read returns a full YAML document.
269 func (r *YAMLReader) Read() ([]byte, error) {
270 var buffer bytes.Buffer
272 line, err := r.reader.Read()
273 if err != nil && err != io.EOF {
277 sep := len([]byte(separator))
278 if i := bytes.Index(line, []byte(separator)); i == 0 {
279 // We have a potential document terminator
282 if len(strings.TrimRightFunc(string(after), unicode.IsSpace)) == 0 {
283 if buffer.Len() != 0 {
284 return buffer.Bytes(), nil
292 if buffer.Len() != 0 {
293 // If we're at EOF, we have a final, non-terminated line. Return it.
294 return buffer.Bytes(), nil
302 type LineReader struct {
306 // Read returns a single line (with '\n' ended) from the underlying reader.
307 // An error is returned iff there is an error with the underlying reader.
308 func (r *LineReader) Read() ([]byte, error) {
316 for isPrefix && err == nil {
317 line, isPrefix, err = r.reader.ReadLine()
320 buffer.WriteByte('\n')
321 return buffer.Bytes(), err
324 // GuessJSONStream scans the provided reader up to size, looking
325 // for an open brace indicating this is JSON. It will return the
326 // bufio.Reader it creates for the consumer.
327 func GuessJSONStream(r io.Reader, size int) (io.Reader, []byte, bool) {
328 buffer := bufio.NewReaderSize(r, size)
329 b, _ := buffer.Peek(size)
330 return buffer, b, hasJSONPrefix(b)
333 var jsonPrefix = []byte("{")
335 // hasJSONPrefix returns true if the provided buffer appears to start with
336 // a JSON open brace.
337 func hasJSONPrefix(buf []byte) bool {
338 return hasPrefix(buf, jsonPrefix)
341 // Return true if the first non-whitespace bytes in buf is
343 func hasPrefix(buf []byte, prefix []byte) bool {
344 trim := bytes.TrimLeftFunc(buf, unicode.IsSpace)
345 return bytes.HasPrefix(trim, prefix)