Application: gvSIG desktop

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA 02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.csv;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.nio.charset.Charset;

import java.util.Locale;

import java.util.Map;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.io.input.BOMInputStream;

import org.apache.commons.lang3.ArrayUtils;

import org.apache.commons.lang3.StringUtils;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.EditableFeatureType;

import static org.gvsig.fmap.dal.store.csv.CSVStoreProvider.NAME;

import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReaderSuperCSV;

import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.JSonReader;

import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader;

import org.gvsig.fmap.geom.Geometry;

import org.gvsig.fmap.geom.GeometryLocator;

import org.gvsig.fmap.geom.GeometryManager;

import org.gvsig.fmap.geom.type.GeometryType;

import org.gvsig.tools.dynobject.Tags;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

/**

 *

 * @author gvSIG  Team

 */

@SuppressWarnings("UseSpecificCatch")

public class CSVUtils {

    private static final Logger LOGGER = LoggerFactory.getLogger(CSVUtils.class);

    private CSVUtils() {

    }

    public static InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException {

        String fullFileName = f==null? "NULL":f.getAbsolutePath();

        Charset charset = Charset.defaultCharset();

        InputStream fis = new BOMInputStream(new FileInputStream(f));

        if (StringUtils.isNotBlank(charsetName)) {

            if (Charset.isSupported(charsetName)) {

                try {

                    charset = Charset.forName(charsetName);

                } catch (Throwable th) {

                    LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + fullFileName + "'.", th);

                }

            } else {

                LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + fullFileName + "'.");

            }

        }

        InputStreamReader isr = new InputStreamReader(fis, charset);

        return isr;

    }

    public static boolean loadFeatureType(CSVStoreParameters parameters, EditableFeatureType featureType, boolean  detectTypes, SimpleTaskStatus status) throws IOException {

        InputStreamReader in = null;

        SimpleReader reader = null;

        try {

            String headers[];

            in = openFile(

                    parameters.getFile(),

                    CSVStoreParameters.getCharset(parameters)

            );

            reader = getSimpleReader(parameters, in);

            headers = CSVStoreParameters.getHeaders(parameters);

            if (headers == null) {

                if (CSVStoreParameters.isFirstLineHeader(parameters)) {

                    headers = reader.getHeader();

                    if (headers == null) {

                        if (CSVStoreParameters.getIgnoreErrors(parameters)) {

                            headers = getFixedHeaders(reader.getColumnsCount());

                        } else {

                            String msg = "Can't retrieve header from csv file '"

                                    + parameters.getFile()

                                            .getAbsolutePath()

                                    + "' and not specified in the parameters.";

                            LOGGER.warn(msg);

                            throw new RuntimeException(msg);

                        }

                    }

                } else {

                    headers = getFixedHeaders(reader.getColumnsCount());

                }

            } else {

                if (CSVStoreParameters.isFirstLineHeader(parameters)) {

                    reader.getHeader(); // Skip and ignore the header of file

                }

            }

            AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null;

            if( detectTypes ) {

                detectedTypes = automaticDetectionOfTypes(parameters, headers, status);

            }

            if( StringUtils.isBlank(headers[headers.length-1]) &&

                (detectedTypes==null || detectedTypes[headers.length-1].isBlank()) ) {

                headers = ArrayUtils.remove(headers, headers.length-1);

            }

            if (detectedTypes != null && detectedTypes.length > headers.length) {

                // Se han detectado mas columnas que las que hay en la cabezera,

                // a?adimos mas columnas a la cabezera.

                String[] headers2 = new String[detectedTypes.length];

                for (int i = 0; i < headers2.length; i++) {

                    if (i < headers.length) {

                        headers2[i] = headers[i];

                    } else {

                        headers2[i] = getFixedHeader(i);

                    }

                }

                headers = headers2;

            }

            for (int i = 0; i < headers.length; i++) {

                if (StringUtils.isBlank(headers[i])) {

                    headers[i] = getFixedHeader(i);

                }

            }

            // Initialize the feature types

            return fillFeatureType(parameters, featureType,  headers, detectedTypes);

        } finally {

            IOUtils.closeQuietly(in);

            IOUtils.closeQuietly(reader);

        }

    }

    public static SimpleReader getSimpleReader(CSVStoreParameters parameters, Reader in) throws IOException {

        SimpleReader reader;

        String filename = CSVStoreParameters.getFileName(parameters);

        if (FilenameUtils.isExtension(filename, "json")){

            reader= new JSonReader(in,parameters);

        } else if (CSVStoreParameters.getRawFieldsDefinition(parameters) != null) {

            reader = new FixedLenReader(in, parameters);

        } else {

            reader = new CSVReaderSuperCSV(in, parameters);

        }

        return reader;

    }

    private static String getFixedHeader(int column) {

        char[] header = new char[3];

        String s = String.format("%03d", column);

        header[0] = (char) (s.charAt(0) + 17);

        header[1] = (char) (s.charAt(1) + 17);

        header[2] = (char) (s.charAt(2) + 17);

        return String.valueOf(header);

    }

    private static String[] getFixedHeaders(int count) {

        String[] headers = new String[count];

        for (int i = 0; i < headers.length; i++) {

            headers[i] = getFixedHeader(i);

        }

        return headers;

    }

    private static AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(CSVStoreParameters parameters, String[] headers, SimpleTaskStatus status) throws IOException {

        String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath();

        boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(parameters);

        if (!automatic_types_detection) {

            return null;

        }

        AutomaticDetectionOfTypes.DetectedValue[] types = null;

        Reader in = null;

        SimpleReader reader = null;

        try {

            in = openFile(

                    parameters.getFile(),

                    CSVStoreParameters.getCharset(parameters)

            );

            reader = getSimpleReader(parameters, in);

            AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes(

                    fullFileName

            );

            types = x.detect(

                    headers.length,

                    reader,

                    CSVStoreParameters.isFirstLineHeader(parameters),

                    CSVStoreParameters.getLocale(parameters),

                    status

            );

        } catch (Exception ex) {

            int lineno = 0;

            if (reader != null) {

                lineno = reader.getLine();

            }

            throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex);

        } finally {

            IOUtils.closeQuietly(reader);

            IOUtils.closeQuietly(in);

        }

        return types;

    }

    private static boolean fillFeatureType(CSVStoreParameters parameters, EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) {

        String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath();

        String providerName = NAME;

        fType.setHasOID(true);

        FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length];

        //

        // Calculamos cuales pueden ser los tipos de datos

        //

        for (int i = 0; i < fieldTypes.length; i++) {

            fieldTypes[i] = new FieldTypeParser(providerName, fullFileName);

        }

        // Asuminos los tipos pasados por parametro, que se supone

        // son los detectados automaticamente.

        if (automaticTypes != null) {

            for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) {

                fieldTypes[i].detectedValue = automaticTypes[i];

                fieldTypes[i].type = automaticTypes[i].getType();

            }

        }

        // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo

        // los tipos anteriores en caso de definirse en la cabezara.

        boolean all_fields_declare_type = true;

        for (int i = 0; i < fieldTypes.length; i++) {

            if (!fieldTypes[i].parse(headers[i])) {

                LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+providerName+"' file '" + fullFileName + "'.");

            }

            if( fieldTypes[i].type == DataTypes.UNKNOWN ) {

                all_fields_declare_type = false;

                fieldTypes[i].type = DataTypes.STRING;

            }

        }

        // Y por ultimo hacemos caso a lo que se haya especificado en los parametros

        // de apertura del CSV, teniendo esto prioridad sobre todo.

        String param_types_def = CSVStoreParameters.getRawFieldTypes(parameters);

        if (StringUtils.isNotBlank(param_types_def)) {

            String sep = CSVStoreParameters.getDelimiter(param_types_def);

            if (StringUtils.isNotBlank(sep)) {

                String[] param_types = param_types_def.split(sep);

                FieldTypeParser parser = new FieldTypeParser(providerName, fullFileName);

                for (String param_type : param_types) {

                    parser.clear();

                    parser.parse(param_type);

                    for (FieldTypeParser fieldType : fieldTypes) {

                        if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {

                            fieldType.copyFrom(parser);

                            break;

                        }

                    }

                }

            }

        }

        //

        // Una vez ya sabemos los tipos de datos rellenamos el feature-type

        //

        Tags ftypeTags = fType.getTags();

        for (FieldTypeParser fieldType : fieldTypes) {

            EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type);

            if( fieldType.detectedValue!=null ) {

                fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));

                fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size));

                if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) {

                    fad.setPrecision(fieldType.detectedValue.getPrecision());

                }

                if( fad.getScale()<fieldType.detectedValue.getScale()) {

                    fad.setScale(fieldType.detectedValue.getScale());

                }

            } else {

                fad.setDisplaySize(fieldType.size);

            }

            if (fieldType.type == DataTypes.GEOMETRY ) {

                fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype);

                if( fType.getDefaultGeometryAttributeName() == null ) {

                    fType.setDefaultGeometryAttributeName(fieldType.name);

                }

            } 

            Locale locale = null;

            if (fieldType.type == DataTypes.TIMESTAMP ) {

                if(!CSVStoreParameters.isBlankOrDefaultLocale(parameters)){

                    locale = CSVStoreParameters.getLocale(parameters);

                }

            } else {

                locale = CSVStoreParameters.getLocale(parameters);

            }

            fad.setLocale(locale);

            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {

                try {

                    switch(entry.getKey().toLowerCase()) {

                        case "expression":

                            // Los campos calculados los procesamos en una segunda

                            // pasada, cuando ya estan definidos el resto de los campos

                            // ya que pueden requerir campos que aun no se han definido.

                            break;

                        default:

                                fad.set(entry.getKey(), entry.getValue());

                            }

                } catch (Exception ex) {

                    LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex);

                }

            }            

            Tags tags = fad.getTags();

            for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) {

                tags.set(entry.getKey(), entry.getValue());

            }

            for (Map.Entry<String, String> entry : fieldType.typetags.entrySet()) {

                ftypeTags.set(entry.getKey(), entry.getValue());

            }

            for (Map.Entry<String, String> entry : fieldType.typeAssignments.entrySet()) {

                try {

                    fType.set(entry.getKey(), entry.getValue());

                } catch(Exception ex) {

                    LOGGER.warn("Can't set attribute '"+entry.getKey()+"' in the feature type.", ex);

                }

            }

        }

        // Processamos ahora los campos calculados

        for (FieldTypeParser fieldType : fieldTypes) {

            EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name);

            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {

                try {

                    switch(entry.getKey().toLowerCase()) {

                        case "expression":

                            fad.set(entry.getKey(), entry.getValue());

                            break;

                    }

                } catch (Exception ex) {

                    LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex);

                }

            }

        }

        String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(parameters);

        if ( pointDimensionNames != null ) {

            CSVPointAttributeEmulator emulator = new CSVPointAttributeEmulator(pointDimensionNames);

            String columnName = CSVStoreParameters.getPointColumnName(parameters);

            if( StringUtils.isBlank(columnName) ) {

                columnName = "geom";

            }

            EditableFeatureAttributeDescriptor attr = fType.add(columnName, DataTypes.GEOMETRY, emulator);

            GeometryManager geommgr = GeometryLocator.getGeometryManager();

            GeometryType gt;

            try {

                if ( emulator.getFieldNames() != null && emulator.getFieldNames().length <= 2 ) {

                        gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM2D);

                } else {

                        gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D);

                }

                attr.setGeometryType(gt);

            } catch (Exception e) {

                LOGGER.warn("Can't set geometry type for the calculated field in '"+providerName+"' file '" + fullFileName + "'.", e);

            }

        }        

        String geometry_column = CSVStoreParameters.getGeometryColumn(parameters);

        if (!StringUtils.isEmpty(geometry_column)) {

            EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column);

            if (attr != null ) {

                if( attr.getType() != DataTypes.GEOMETRY ) {

                    attr.setDataType(DataTypes.GEOMETRY);

                }

                GeometryManager geommgr = GeometryLocator.getGeometryManager();

                GeometryType gt;

                try {

                    gt = geommgr.getGeometryType(

                            CSVStoreParameters.getGeometryType(parameters),

                            CSVStoreParameters.getGeometrySubType(parameters)

                    );

                    attr.setGeometryType(gt);

                } catch (Exception e) {

                    LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + fullFileName + "'.", e);

                }

                fType.setDefaultGeometryAttributeName(geometry_column);

            }

        }

        return all_fields_declare_type;

    }

}