CsvUtil.qm.dox.h

// -*- mode: c++; indent-tabs-mode: nil -*-
// @file CsvUtil.qm Qore user module for working with CSV files

/*  CsvUtil.qm Copyright 2012 - 2016 Qore Technologies, sro

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

// minimum required Qore version

// assume local var scope, do not use "$" for vars, members, and method calls


/* see release notes below for version history
*/

class CsvHelper {

public:
    private :
        const C_OPT1 = 0x1;
        const C_OPT2 = 0x2;
        const Types = (
            "int": True,
            "*int": True,
            "float": True,
            "*float": True,
            "number": True,
            "*number": True,
            "string": True,
            "*string": True,
            "date": True,
            "*date": True,
        );

        const FieldAttrs = ("type", "format", "timezone", "code", "header");

        bool tolwr = False;

        string date_format;

        hash m_specs;

        string errname;

        // reorder data according headers set by options.headers or read from CsvHeader
        bool headerReorder = True;


public:

    constructor (string n_errname);


    private bool isMultiType();


    private checkType(string fld_errs, string key, string value);


    // get spec from options.fields for old Csv. Check spec param for new Csv
    private hash getSpec(*hash fields, string fld_errs, int C_OPTx);


    private hash getSpec1(*hash fields);


    private hash getSpec2(hash spec);


    private list adjustFieldsFromHeaders(string type, *list headers);


}; // class CsvHelper

namespace CsvUtil {
    const EOL_UNIX = "\n";
    const EOL_WIN = "\r\n";
    const EOL_MACINTOSH = "\r";

    // helper list of end of line values
    const EOLS = (EOL_UNIX, EOL_WIN, EOL_MACINTOSH, );

    const CSV_TYPE_UNKNOWN = "<unknown>";
    const CSV_TYPE_SINGLE = "<single>";



class AbstractCsvIterator : public Qore::AbstractIterator, private CsvHelper {

public:
        private :
            const Options = (
                "date_format": C_OPT1|C_OPT2,
                "date-format": C_OPT1|C_OPT2,
                "encoding": C_OPT1|C_OPT2,
                "eol": C_OPT1|C_OPT2,
                "extended_record": C_OPT2,
                "fields": C_OPT1,
                "header-lines": C_OPT1|C_OPT2,
                "header_lines": C_OPT1|C_OPT2,
                "header-names": C_OPT1|C_OPT2,
                "header_names": C_OPT1|C_OPT2,
                "header_reorder": C_OPT1|C_OPT2,
                "headers": C_OPT1,
                "ignore-empty": C_OPT1|C_OPT2,
                "ignore_empty": C_OPT1|C_OPT2,
                "ignore-whitespace": C_OPT1|C_OPT2,
                "ignore_whitespace": C_OPT1|C_OPT2,
                "quote": C_OPT1|C_OPT2,
                "separator": C_OPT1|C_OPT2,
                "timezone": C_OPT1|C_OPT2,
                "tolwr": C_OPT1|C_OPT2,
                "verify-columns": C_OPT1|C_OPT2,
                "verify_columns": C_OPT1|C_OPT2,
                );

            // field separator
            string separator = ",";

            // field content delimiter
            string quote = "\"";

            // number of header lines
            softint headerLines = 0;

            // flag to use string names from the first header row if possible
            bool headerNames = False;

            // True if empty lines should be ignored
            bool ignoreEmptyLines = True;

            // Flag to trim the field content (trim leading and trailing whitespace) from unquoted fields
            bool ignoreWhitespace = True;

            // the @ref Qore::TimeZone to use when parsing dates (default: current time zone)
            *TimeZone timezone;

            // verify the column count for every row; if a row does not match, then throw a \c CSVFILEITERATOR-DATA-ERROR exception
            bool checkElementCounts = False;

            // getRecord/getValue returns extended hash
            bool extendedRecord = False;

            // column count for verifying column counts
            int cc;

            // current record count for the index() method
            int rc = 0;

            // to resolve record type by rules
            hash m_resolve_by_rule;

            // to resolve record type by number of fields
            hash m_resolve_by_count;

            // list of idx to field transformarions, in order of spec
            hash m_resolve_by_idx;

            // fake specs based on the first non-header row
            bool fakeHeaderNames;


public:


        constructor(*hash opts);



        // NOTE: when declared as *hash then always calls this constructor
        constructor(hash spec, hash opts);


        private processCommonOptions(*hash opts, int C_OPTx);


        private processSpec(hash spec);


        private prepareFieldsFromHeaders(*list headers);


        private *string getDataName();


        private abstract int lineNumberImpl();

        private abstract string getLineValueImpl();

        private abstract bool nextLineImpl();


        bool next();



        any memberGate(string name);



        hash getValue();



        hash getRecord(bool extended);



        hash getRecord();



        any getRecordList();



        string getSeparator();



        string getQuote();



        *list getHeaders();



        *list getHeaders(string type);



        int index();



        int lineNumber();


        private any handleType(hash fh, *string val);


        private list getLineAndSplit();



        string identifyType(list rec);



        private *string identifyTypeImpl(list rec);


        private hash parseLine();

    };


class CsvFileIterator : public CsvUtil::AbstractCsvIterator,public  Qore::FileLineIterator {

public:

        constructor(string path, *hash opts) ;


        constructor(string path, hash spec, hash opts) ;


        any memberGate(string name);


        private *string getDataName();


        private int lineNumberImpl();


        private string getLineValueImpl();


        private bool nextLineImpl();

    }; // CsvFileIterator class


class CsvDataIterator : public CsvUtil::AbstractCsvIterator,public  DataLineIterator {

public:


        constructor(string data, *hash opts) ;



        constructor(string data, hash spec, hash opts) ;


        any memberGate(string name);


        private int lineNumberImpl();


        private string getLineValueImpl();


        private bool nextLineImpl();

    };


class AbstractCsvWriter : private CsvHelper {

public:
        private :
            const Options = (
                "block": C_OPT1|C_OPT2,
                "datamap": C_OPT1,
                "date_format": C_OPT1|C_OPT2,
                "date-format": C_OPT1|C_OPT2,
                "encoding": C_OPT1|C_OPT2,
                "eol": C_OPT1|C_OPT2,
                "fields": C_OPT1,
                "headers": C_OPT1,
                "header_reorder": C_OPT1,
                "info_log": C_OPT1|C_OPT2,
                "optimal_quotes": C_OPT1|C_OPT2,
                "optimal-quotes": C_OPT1|C_OPT2,
                "quote": C_OPT1|C_OPT2,
                "quote_escape": C_OPT1|C_OPT2,
                "separator": C_OPT1|C_OPT2,
                "verify_columns": C_OPT1|C_OPT2,
                "verify-columns": C_OPT1|C_OPT2,
                "write_headers": C_OPT1|C_OPT2,
                "write-headers": C_OPT1|C_OPT2,
                );

            string encoding;

            string separator = ",";

            string quote = "\"";

            string m_quoteEscapeChar = "\\";

            string eol = EOL_UNIX;

            bool checkElementCounts = False;

            int lineNo = 0;

            int block = 1000;

            string baseTemplate;

            bool write_headers = True;

            bool optimal_quotes = True;

            *code info_log;

            hash m_out_by_name;

            hash m_out_by_idx;

public:


        constructor(string n_errname, *hash n_opts);



        constructor(string n_errname, hash spec, hash n_opts);


        private processCommonOptions(*hash n_opts, int C_OPTx);


        private processSpec();


        private writeHeaders();



        writeLine(list values);



        writeLine(hash values);



        writeLine(string type, list values);



        writeLine(string type, hash values);



        write(Qore::AbstractIterator iterator);



        write(Qore::SQL::SQLStatement iterator);



        write(list l);


        abstract private writeRawLine(list values);


        private string prepareRawLine(list values);


    }; // AbstractCsvWriter class

class CsvFileWriter : public AbstractCsvWriter {

public:

        private :
            // a file to write
            File file;

public:


        constructor(string path, *hash opts) ;



        constructor(string path, hash spec, hash opts) ;


        private openFile(string path);


        private writeRawLine(list values);


    }; // CsvFileWriter

class CsvStringWriter : public AbstractCsvWriter {

public:

        private :
            // a csv content
            string content;

public:


        constructor(*hash opts) ;



        constructor(hash spec, hash opts) ;


        private initContent();


        private writeRawLine(list values);



        string write(Qore::AbstractIterator iterator);



        string write(list l);


        string getContent();


    }; // CsvStringWriter

}; // CsvUtil namespace