From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- modules/csv/csv.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 modules/csv/csv.go (limited to 'modules/csv/csv.go') diff --git a/modules/csv/csv.go b/modules/csv/csv.go new file mode 100644 index 0000000..35c5d6a --- /dev/null +++ b/modules/csv/csv.go @@ -0,0 +1,149 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package csv + +import ( + "bytes" + stdcsv "encoding/csv" + "io" + "path/filepath" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/translation" + "code.gitea.io/gitea/modules/util" +) + +const ( + maxLines = 10 + guessSampleSize = 1e4 // 10k +) + +// CreateReader creates a csv.Reader with the given delimiter. +func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { + rd := stdcsv.NewReader(input) + rd.Comma = delimiter + if delimiter != '\t' && delimiter != ' ' { + // TrimLeadingSpace can't be true when delimiter is a tab or a space as the value for a column might be empty, + // thus would change `\t\t` to just `\t` or ` ` (two spaces) to just ` ` (single space) + rd.TrimLeadingSpace = true + } + return rd +} + +// CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. +// Reads at most guessSampleSize bytes. +func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) { + data := make([]byte, guessSampleSize) + size, err := util.ReadAtMost(rd, data) + if err != nil { + return nil, err + } + + return CreateReader( + io.MultiReader(bytes.NewReader(data[:size]), rd), + determineDelimiter(ctx, data[:size]), + ), nil +} + +// determineDelimiter takes a RenderContext and if it isn't nil and the Filename has an extension that specifies the delimiter, +// it is used as the delimiter. Otherwise we call guessDelimiter with the data passed +func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { + extension := ".csv" + if ctx != nil { + extension = strings.ToLower(filepath.Ext(ctx.RelativePath)) + } + + var delimiter rune + switch extension { + case ".tsv": + delimiter = '\t' + case ".psv": + delimiter = '|' + default: + delimiter = guessDelimiter(data) + } + + return delimiter +} + +// quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a +// field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt +// This finds all quoted strings that have escaped quotes. +var quoteRegexp = regexp.MustCompile(`"[^"]*"`) + +// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can reliably have each row on one line +// (quoted strings often have new lines within the string) +func removeQuotedString(text string) string { + return quoteRegexp.ReplaceAllLiteralString(text, "") +} + +// guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. +// If more than one delimiter passes, the delimiter that results in the most columns is returned. +func guessDelimiter(data []byte) rune { + delimiter := guessFromBeforeAfterQuotes(data) + if delimiter != 0 { + return delimiter + } + + // Removes quoted values so we don't have columns with new lines in them + text := removeQuotedString(string(data)) + + // Make the text just be maxLines or less, ignoring truncated lines + lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines + if len(lines) > maxLines { + // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines + lines = lines[:maxLines] + } else if len(lines) > 1 && len(data) >= guessSampleSize { + // Even with data >= guessSampleSize, we don't have maxLines + 1 (no extra lines, must have really long lines) + // thus the last line is probably have a truncated line. Drop the last line if len(lines) > 1 + lines = lines[:len(lines)-1] + } + + // Put lines back together as a string + text = strings.Join(lines, "\n") + + delimiters := []rune{',', '\t', ';', '|', '@'} + validDelim := delimiters[0] + validDelimColCount := 0 + for _, delim := range delimiters { + csvReader := stdcsv.NewReader(strings.NewReader(text)) + csvReader.Comma = delim + if rows, err := csvReader.ReadAll(); err == nil && len(rows) > 0 && len(rows[0]) > validDelimColCount { + validDelim = delim + validDelimColCount = len(rows[0]) + } + } + return validDelim +} + +// FormatError converts csv errors into readable messages. +func FormatError(err error, locale translation.Locale) (string, error) { + if perr, ok := err.(*stdcsv.ParseError); ok { + if perr.Err == stdcsv.ErrFieldCount { + return locale.TrString("repo.error.csv.invalid_field_count", perr.Line), nil + } + return locale.TrString("repo.error.csv.unexpected", perr.Line, perr.Column), nil + } + + return "", err +} + +// Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes +var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t;|]{0,1})`) + +// guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote, +// or a double quote with a closing quote and a valid delimiter after it +func guessFromBeforeAfterQuotes(data []byte) rune { + rs := beforeAfterQuotes.FindStringSubmatch(string(data)) // returns first match, or nil if none + if rs != nil { + if rs[1] != "" { + return rune(rs[1][0]) // delimiter found left of quoted string + } else if rs[2] != "" { + return rune(rs[2][0]) // delimiter found right of quoted string + } + } + return 0 // no match found +} -- cgit v1.2.3