Application: gvSIG desktop

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or modify it under

 * the terms of the GNU General Public License as published by the Free Software

 * Foundation; either version 3 of the License, or (at your option) any later

 * version.

 *

 * This program is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more

 * details.

 *

 * You should have received a copy of the GNU General Public License along with

 * this program; if not, write to the Free Software Foundation, Inc., 51

 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us at info AT

 * gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.simplereader;

import java.io.Closeable;

import java.io.File;

import java.io.IOException;

import java.io.InputStreamReader;

import java.io.Reader;

import java.net.URI;

import java.net.URL;

import java.nio.charset.StandardCharsets;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.Iterator;

import java.util.List;

import java.util.Map;

import java.util.Objects;

import org.apache.commons.io.FileUtils;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.cresques.cts.IProjection;

import org.gvsig.fmap.dal.DALLocator;

import org.gvsig.fmap.dal.DataManager;

import org.gvsig.fmap.dal.DataServerExplorer;

import org.gvsig.fmap.dal.DataStore;

import org.gvsig.fmap.dal.DataStoreNotification;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.exception.CloseException;

import org.gvsig.fmap.dal.exception.DataException;

import org.gvsig.fmap.dal.exception.InitializeException;

import org.gvsig.fmap.dal.exception.OpenException;

import org.gvsig.fmap.dal.exception.ReadException;

import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.EditableFeatureType;

import org.gvsig.fmap.dal.feature.Feature;

import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.FeatureQuery;

import org.gvsig.fmap.dal.feature.FeatureStore;

import org.gvsig.fmap.dal.feature.FeatureType;

import org.gvsig.fmap.dal.feature.exception.CreateFeatureException;

import org.gvsig.fmap.dal.feature.exception.PerformEditingException;

import org.gvsig.fmap.dal.feature.spi.FeatureProvider;

import org.gvsig.fmap.dal.feature.spi.FeatureReferenceProviderServices;

import org.gvsig.fmap.dal.feature.spi.FeatureSetProvider;

import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider;

import org.gvsig.fmap.dal.resource.file.FileResource;

import org.gvsig.fmap.dal.resource.spi.ResourceConsumer;

import org.gvsig.fmap.dal.resource.spi.ResourceProvider;

import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer;

import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters;

import org.gvsig.fmap.dal.spi.DataStoreProviderServices;

import org.gvsig.fmap.dal.store.simplereader.simplereaders.AbstractSimpleReader;

import org.gvsig.fmap.dal.store.simplereader.simplereaders.SimpleReader;

import org.gvsig.fmap.geom.Geometry;

import org.gvsig.fmap.geom.GeometryLocator;

import org.gvsig.fmap.geom.GeometryManager;

import org.gvsig.fmap.geom.SpatialIndex;

import org.gvsig.fmap.geom.SpatialIndexFactory;

import org.gvsig.fmap.geom.primitive.Envelope;

import org.gvsig.fmap.geom.primitive.Point;

import org.gvsig.tools.ToolsLocator;

import org.gvsig.tools.dataTypes.Coercion;

import org.gvsig.tools.dataTypes.CoercionContext;

import org.gvsig.tools.dispose.DisposableIterator;

import org.gvsig.tools.dispose.DisposeUtils;

import org.gvsig.tools.dynobject.DynObject;

import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException;

import org.gvsig.tools.evaluator.AbstractEvaluator;

import org.gvsig.tools.evaluator.EvaluatorData;

import org.gvsig.tools.evaluator.EvaluatorException;

import org.gvsig.tools.exception.BaseException;

import org.gvsig.tools.exception.NotYetImplemented;

import org.gvsig.tools.i18n.I18nManager;

import org.gvsig.tools.persistence.PersistentState;

import org.gvsig.tools.persistence.exception.PersistenceException;

import org.gvsig.tools.task.SimpleTaskStatus;

import org.gvsig.tools.task.TaskStatusManager;

import org.gvsig.tools.task.UserCancelTaskException;

import org.gvsig.tools.util.GetItemWithSize64;

import org.gvsig.tools.visitor.VisitCanceledException;

import org.gvsig.tools.visitor.Visitor;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

@SuppressWarnings("UseSpecificCatch")

public abstract class SimpleReaderStoreProvider extends AbstractMemoryStoreProvider implements

        ResourceConsumer {

    protected static final Logger LOGGER = LoggerFactory.getLogger(SimpleReaderStoreProvider.class);

//    public static final String NAME = DataStore.CSV_PROVIDER_NAME;

//    public static final String DESCRIPTION = "CSV file";

//

//    public static final String METADATA_DEFINITION_NAME = NAME;

    protected final ResourceProvider resource;

    protected long counterNewsOIDs = 0;

    protected Map<String,Envelope> envelopes;

    protected boolean need_calculate_envelope = false;

    protected final SimpleTaskStatus taskStatus;

    protected FeatureType featureType;

    protected GetItemWithSize64<List<String>> virtualrows;

    protected RowToFeatureTranslator rowToFeatureTranslator;

    protected Map<String,SpatialIndex> spatialIndexes;

    @SuppressWarnings({"OverridableMethodCallInConstructor", "LeakingThisInConstructor"})

    public SimpleReaderStoreProvider(

            SimpleReaderStoreParameters parameters,

            DataStoreProviderServices storeServices,

            DynObject metadata

        ) throws InitializeException {

        super(

                parameters,

                storeServices,

                metadata

        );

        TaskStatusManager manager = ToolsLocator.getTaskStatusManager();

        this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV");

        this.taskStatus.setAutoremove(true);

        this.envelopes = new HashMap<>();

        counterNewsOIDs = 0;

        File file = getSimpleReaderParameters().getFile();

        resource = this.createResource(

                FileResource.NAME,

                new Object[]{file.getAbsolutePath()}

        );

        resource.addConsumer(this);

        initializeFeatureTypes();

    }

    private SimpleReaderStoreParameters getSimpleReaderParameters() {

        return (SimpleReaderStoreParameters) this.getParameters();

    }

    @Override

    public abstract String getProviderName();

    @Override

    public boolean allowWrite() {

        return false;

    }

    protected String getFullFileName() {

        // Usar solo para mostrar mensajes en el logger.

        String s;

        try {

            s = getSimpleReaderParameters().getFile().getAbsolutePath();

        } catch (Exception e2) {

            s = "(unknow)";

        }

        return s;

    }

    @Override

    public void open() throws OpenException {

        if (this.data != null) {

            return;

        }

        this.data = new ArrayList<>();

        resource.setData(new HashMap());

        counterNewsOIDs = 0;

        try {

            loadFeatures();

        } catch (RuntimeException e) {

            LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e);

            throw e;

        } catch (Exception e) {

            LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e);

            throw new RuntimeException(e);

        }

    }

    @Override

    public DataServerExplorer getExplorer() throws ReadException {

        DataManager manager = DALLocator.getDataManager();

        FilesystemServerExplorerParameters params;

        try {

            params = (FilesystemServerExplorerParameters) manager

                    .createServerExplorerParameters(FilesystemServerExplorer.NAME);

            params.setRoot(this.getSimpleReaderParameters().getFile().getParent());

            return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);

        } catch (Exception e) {

            throw new ReadException(this.getProviderName(), e);

        }

    }

    @Override

    @SuppressWarnings("Convert2Lambda")

    public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException {

        throw new UnsupportedOperationException();

    }

    @Override

    public boolean closeResourceRequested(ResourceProvider resource) {

        return true;

    }

    @Override

    public int getOIDType() {

        return DataTypes.LONG;

    }

    @Override

    public boolean supportsAppendMode() {

        return false;

    }

    @Override

    @SuppressWarnings("Convert2Lambda")

    public void append(final FeatureProvider featureProvider) {

        throw new UnsupportedOperationException();

    }

    @Override

    @SuppressWarnings("Convert2Lambda")

    public void beginAppend() throws DataException {

        throw new UnsupportedOperationException();

    }

    @Override

    @SuppressWarnings("Convert2Lambda")

    public void endAppend() {

        throw new UnsupportedOperationException();

    }

    public void saveToState(PersistentState state) throws PersistenceException {

        throw new NotYetImplemented();

    }

    public void loadFromState(PersistentState state) throws PersistenceException {

        throw new NotYetImplemented();

    }

    @Override

    public Object createNewOID() {

        return counterNewsOIDs++;

    }

    protected void initializeFeatureTypes() throws InitializeException {

        try {

            this.open();

        } catch (OpenException e) {

            throw new InitializeException(this.getProviderName(), e);

        }

    }

    @Override

    @SuppressWarnings("Convert2Lambda")

    public Envelope getEnvelope() throws DataException {

        this.open();

        FeatureAttributeDescriptor geomdesc = this.featureType.getDefaultGeometryAttribute();

        String geomName = geomdesc.getName();

        Envelope env = this.envelopes.get(geomName);

        if (env != null) {

            return env;

        }

        env = bboxFileLoad("_"+geomName);

        if (env != null) {

            this.envelopes.put(geomName, env);

            return env;

        }

        if (!this.need_calculate_envelope) {

            return null;

        }

        try {

            I18nManager i18n = ToolsLocator.getI18nManager();

            this.taskStatus.add();

            this.taskStatus.message(i18n.getTranslation("_Calculating_envelope"));

            FeatureStore fs = this.getFeatureStore();

            FeatureType ft = fs.getDefaultFeatureType();

            FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex());

            this.taskStatus.setRangeOfValues(0, fs.getFeatureCount());

            env = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());

            fs.accept(new Visitor() {

                @Override

                public void visit(Object obj) throws VisitCanceledException, BaseException {

                    taskStatus.incrementCurrentValue();

                    if(taskStatus.isCancellationRequested()){

                        taskStatus.cancel();

                        throw new VisitCanceledException();

                    }

                    Feature f = (Feature) obj;

                    Geometry geom = f.getDefaultGeometry();

                    if (geom != null) {

                        try {

                            Envelope env = geom.getEnvelope();

                            env.add(env);

                        } catch(Exception ex) {

                            LOGGER.warn("Can't calculate envelop of geometry in feature '"+Objects.toString(f.getReference())+"'.",ex);

                        }

                    }

                }

            });

            bboxFileSave("_"+geomName,env);

            taskStatus.terminate();

        } catch (VisitCanceledException e) {

            return null;

        } catch (BaseException e) {

            taskStatus.abort();

            LOGGER.warn("Can't calculate the envelope of file '" + this.getFullName() + "'.", e);

            return null;

        }

        this.need_calculate_envelope = false;

        return env;

    }

    @Override

    public Object getDynValue(String name) throws DynFieldNotFoundException {

        if (DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name)) {

            try {

                return this.getEnvelope();

            } catch (DataException e) {

                return null;

            }

        } else {

            if (DataStore.METADATA_CRS.equalsIgnoreCase(name)) {

                IProjection pro = SimpleReaderStoreParameters.getCRS(this.getSimpleReaderParameters());

                if (pro != null) {

                    return pro;

                }

            }

        }

        return super.getDynValue(name);

    }

    @Override

    public void resourceChanged(ResourceProvider resource) {

        this.getStoreServices().notifyChange(

                DataStoreNotification.RESOURCE_CHANGED,

                resource);

    }

    @Override

    public Object getSourceId() {

        return this.getSimpleReaderParameters().getFile();

    }

    @Override

    public String getName() {

        String name = this.getSimpleReaderParameters().getFile().getName();

        return FilenameUtils.getBaseName(name);

    }

    @Override

    public String getFullName() {

        return this.getSimpleReaderParameters().getFile().getAbsolutePath();

    }

    @Override

    public ResourceProvider getResource() {

        return resource;

    }

    private boolean isEmpty(String s) {

        if (s == null) {

            return true;

        }

        return s.trim().length() == 0;

    }

    private void init(SimpleReaderStoreParameters parameters, DataStoreProviderServices storeServices) {

        throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.

    }

    static class ToPointEvaluaror extends AbstractEvaluator {

        private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);

        private GeometryManager geommgr = null;

        private String xname = null;

        private String yname = null;

        private String zname = null;

        private final Coercion toDouble;

        private int errorcount = 0;

        ToPointEvaluaror(String[] pointDimensionNames) {

            this.xname = pointDimensionNames[0];

            this.yname = pointDimensionNames[1];

            if (pointDimensionNames.length > 2) {

                this.zname = pointDimensionNames[2];

            }

            this.geommgr = GeometryLocator.getGeometryManager();

            this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);

        }

        @Override

        public Object evaluate(EvaluatorData data) throws EvaluatorException {

            try {

                double x = ((Double) toDouble.coerce(data.getDataValue(xname)));

                double y = ((Double) toDouble.coerce(data.getDataValue(yname)));

                Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);

                if (zname != null) {

                    double z = ((Double) toDouble.coerce(data.getDataValue(zname)));

                    point.setCoordinateAt(2, z);

                }

                return point;

            } catch (Exception ex) {

                if (++errorcount < 5) {

                    logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"

                            + xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString());

                }

                return null;

            }

        }

        @Override

        public String getName() {

            return "ToPointEvaluaror";

        }

    }

    public static class RowToFeatureTranslator {

        private Coercion coercion[];

        private CoercionContext coercionContext[];

        private int sizes[];

        private String[] names;

        private final boolean ignore_errors;

        private long count_errors;

        private FeatureType csvFeatureType;

        public RowToFeatureTranslator(boolean ignore_errors) {

            this.ignore_errors = ignore_errors;

            this.count_errors = 0;

        }

        public int getColumnSize(int column) {

            return this.sizes[column];

        }

        public void initialize(FeatureType ftype) {

            this.csvFeatureType = ftype;

            int columns = this.csvFeatureType.size();

            this.names = new String[columns];

            this.coercion = new Coercion[columns];

            this.coercionContext = new CoercionContext[columns];

            this.sizes = new int[columns];

            int index = 0;

            for (int i = 0; i < this.csvFeatureType.size(); i++) {

                FeatureAttributeDescriptor ad = this.csvFeatureType.getAttributeDescriptor(i);

                names[i] = null;

                if( ad.isComputed() ) {

                    continue;

                }

                names[index] = ad.getName();

                coercion[index] = ad.getCoercion();

                coercionContext[index] = ad.getCoercionContext();

                sizes[index] = ad.getSize();

                index++;

            }

        }

        public void translate(long rowindex, List<String> row, FeatureProvider feature) throws Exception {

            feature.setOID(rowindex);

            for (int i = 0; i < names.length; i++) {

                String name = names[i];

                if( name == null ) {

                    break;

                }

                Object rawvalue = row.get(i);

                try {

                    Object value = null;

                    if (coercion[i] != null) {

                        value = coercion[i].coerce(rawvalue, coercionContext[i]);

                    }

                    int findex = feature.getType().getIndex(name);

                    if( findex>=0 ) {

                        // Ojo que puede que se este filtrando el featuretype y no 

                        // tenga todos los atributos, por ejemplo al pintar la vista.

                        feature.set(findex, value);

                    }

                    if (sizes[i] >= 0

                            && (value instanceof String || value instanceof URL

                            || value instanceof URI || value instanceof File)) {

                        int x = value.toString().length();

                        if (sizes[i] < x) {

                            sizes[i] = x;

                        }

                    }

                } catch (Exception ex) {

                    if (!ignore_errors) {

                        throw ex;

                    }

                    if (count_errors++ < 10) {

                        LOGGER.warn("Can't load value of attribute " + name +"/" +i+" in row " + rowindex + ".", ex);

                    }

                    if (count_errors == 10) {

                        LOGGER.info("Too many errors, suppress messages.");

                    }

                }

            }

        }

    }

    protected abstract SimpleReaderFeatureTypeLoader getFeatureTypeLoader();

    protected void loadFeatures() {

        InputStreamReader in = null;

        SimpleReader reader = null;

        try {

            taskStatus.setTitle("CSV "+this.getName());

            taskStatus.add();

//            boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());

            // Initialize the feature types

            EditableFeatureType edftype = getStoreServices().createFeatureType(this.getName());

            SimpleReaderFeatureTypeLoader featureTypeLoader = getFeatureTypeLoader();

            featureTypeLoader.loadFeatureType(edftype, taskStatus);

            FeatureType ftype = edftype.getNotEditableCopy();

            this.setFeatureType(ftype);

            in = SimpleReaderUtils.openFile(

                    this.getSimpleReaderParameters().getFile(),

                    SimpleReaderStoreParameters.getCharset(this.getSimpleReaderParameters())

            );

            reader = getSimpleReader(getSimpleReaderParameters(), in);

            if (featureTypeLoader.isFirstLineHeader()) {

                reader.getHeader(); // Skip and ignore the header of file

            }

            this.rowToFeatureTranslator = new RowToFeatureTranslator(

                    SimpleReaderStoreParameters.getIgnoreErrors(getSimpleReaderParameters())

            );

            this.rowToFeatureTranslator.initialize(ftype);

            if (ftype.getDefaultGeometryAttributeName() != null) {

                this.need_calculate_envelope = true;

            }

            I18nManager i18n = ToolsLocator.getI18nManager();

            taskStatus.message(i18n.getTranslation("_Loading"));

            if(this.virtualrows != null && this.virtualrows instanceof Closeable){

                IOUtils.closeQuietly((Closeable) this.virtualrows);

                this.virtualrows = null;

            }

            this.virtualrows = ((AbstractSimpleReader) reader).getVirtualRows(this.taskStatus);

            if(this.taskStatus.isCancellationRequested()){

                throw new UserCancelTaskException();

            }

            if (this.virtualrows == null) {

                taskStatus.message(i18n.getTranslation("_Loading"));

                taskStatus.setIndeterminate();

                List<String> row = reader.read();

                int skipLines = SimpleReaderStoreParameters.getSkipLines(getSimpleReaderParameters());

                if (skipLines > 0) {

                    row = reader.skip(skipLines);

                }

                int limit = SimpleReaderStoreParameters.getLimit(getSimpleReaderParameters());

                while (row != null) {

                    taskStatus.incrementCurrentValue();

                    if( taskStatus.isCancellationRequested() ) {

                        taskStatus.cancel();

                        break;

                    }

                    FeatureProvider feature = this.createFeatureProvider(ftype);

                    this.rowToFeatureTranslator.translate(reader.getLine(), row, feature);

                    this.addFeatureProvider(feature);

                    if (limit > 0) {

                        if (limit < this.data.size()) {

                            break;

                        }

                    }

                    row = reader.read();

                }

                for (int i = 0; i < ftype.size(); i++) {

                    if (this.rowToFeatureTranslator.getColumnSize(i) > 0) {

//                    if (sizes[i] > 0) {

                        EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i));

//                        efad.setSize(sizes[i]);

                        efad.setSize(this.rowToFeatureTranslator.getColumnSize(i));

                    }

                }

                // Volvemos a asignar al store el featuretype, ya que puede

                // haber cambiado.

                ftype = edftype.getNotEditableCopy();

                this.setFeatureType(ftype);

            } else {

                this.loadOrCreateSpatialIndex();

            }

            if( taskStatus.isRunning() ) {

                taskStatus.terminate();

            }

        } catch (UserCancelTaskException ex) {

            this.taskStatus.cancel();

            throw ex;

        } catch (Throwable ex) {

            taskStatus.abort();

            int lineno = 0;

            if (reader != null) {

                lineno = reader.getLine();

            }

            throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near record " + lineno + ".", ex);

        } finally {

            if (reader != null) {

                try {

                    reader.close();

                } catch (Exception ex) {

                    // Do nothing

                }

//                reader = null;

            }

            if (in != null) {

                try {

                    in.close();

                } catch (Exception ex) {

                    // Do nothing

                }

//                in = null;

            }

            taskStatus.remove();

        }

    }

    @Override

    public void fixFeatureTypeFromParameters() {

        if(mustFixFeatureType() && featureType != null){

            this.setFeatureType(featureType);

        }

    }

    protected boolean mustFixFeatureType() {

        return false;

    }

    private void setFeatureType(FeatureType ftype) {

        try {

            List<FeatureType> ftypes = new ArrayList<>();

            ftypes.add(ftype);

            this.featureType = ftype;

            if(this.getStoreServices().getDefaultFeatureType() == null){

                this.getStoreServices().setFeatureTypes(ftypes, ftype);

                return;

            }

            if(mustFixFeatureType()){

                this.getStoreServices().setFeatureTypes(ftypes, ftype);

            }

        } catch (DataException ex) {

            LOGGER.warn("Cant set feature type", ex);

        }

    }

    @Override

    public FeatureSetProvider createSet(FeatureQuery query, FeatureType providerFeatureType, FeatureType featureType)

            throws DataException {

        this.open();

        if (this.virtualrows == null) {

            return super.createSet(query, providerFeatureType, featureType);

        }

        return new SimpleReaderSetProvider(this, query, providerFeatureType, featureType);

    }

    @Override

    public FeatureSetProvider createSet(FeatureQuery query, FeatureType featureType)

            throws DataException {

        this.open();

        if (this.virtualrows == null) {

            return super.createSet(query, featureType);

        }

        return new SimpleReaderSetProvider(this, query, featureType, featureType);

    }

    public List<String> getRowByIndex(long index) {

        try {

            this.open();

        } catch(Exception ex) {

            throw new RuntimeException("Can't get row by index", ex);

        }

        if (this.virtualrows == null) {

            return null;

        }

        List<String> line = this.virtualrows.get64(index);

        return line;

    }

    public RowToFeatureTranslator getRowToFeatureTranslator() {

        if (this.rowToFeatureTranslator == null) {

            boolean ignore_errors = SimpleReaderStoreParameters.getIgnoreErrors(getSimpleReaderParameters());

            this.rowToFeatureTranslator = new RowToFeatureTranslator(ignore_errors);

            this.rowToFeatureTranslator.initialize(featureType);

        }

        return this.rowToFeatureTranslator;

    }

    @Override

    public long getFeatureCount() throws DataException {

        this.open();

        if (this.virtualrows == null) {

            return super.getFeatureCount();

        }

        return this.virtualrows.size64();

    }

    @Override

    public long getDataSize() throws DataException {

        this.open();

        if (this.virtualrows == null) {

            return super.getDataSize();

        }

        return this.virtualrows.size64();

    }

    @Override

    protected FeatureProvider internalGetFeatureProviderByReference(

            FeatureReferenceProviderServices reference) throws DataException {

        this.open();

        if (this.virtualrows == null) {

            return super.internalGetFeatureProviderByReference(reference);

        }

        int oid = ((Long) reference.getOID()).intValue();

        RowToFeatureTranslator translator = getRowToFeatureTranslator();

        FeatureProvider feature = this.createFeatureProvider(this.featureType);

        try {

            translator.translate(oid, this.virtualrows.get64(oid), feature);

        } catch (Exception ex) {

            throw new CreateFeatureException(ex, this.getName());

        }

        return feature;

    }

    @Override

    protected void doDispose() throws BaseException {

        super.doDispose();

        if (this.virtualrows != null && this.virtualrows instanceof Closeable) {

            IOUtils.closeQuietly((Closeable) this.virtualrows);

            this.virtualrows = null;

        }

    }

    @Override

     public void refresh() throws OpenException {

        try {

            this.close();

        } catch (CloseException e) {

            throw new OpenException(this.getProviderName(), e);

        }

        this.open();

    }

    @Override

    public void close() throws CloseException {

        super.close(); //To change body of generated methods, choose Tools | Templates.

        this.data = null;

        if(this.virtualrows != null && this.virtualrows instanceof Closeable){

            IOUtils.closeQuietly((Closeable) this.virtualrows);

            this.virtualrows = null;

            this.envelopes = null;

            this.spatialIndexes = null;

        }

    }

    private void loadOrCreateSpatialIndex() {

        FeatureSetProvider set = null;

        DisposableIterator<FeatureProvider> it = null;

        try {

            if (this.virtualrows == null) {

                return;

            }

            this.spatialIndexes = new HashMap<>();

            for (FeatureAttributeDescriptor geomdesc : this.featureType) {

                if (geomdesc.getType() != DataTypes.GEOMETRY) {

                    continue;

                }

                String indexTypeName = GeometryManager.SPATIALINDEX_DEFAULT_QUADTREE;

                String extname = "qtree";

                String geomName = geomdesc.getName();

                Envelope env = bboxFileLoad("_"+geomName);

                File indexfile = this.getAuxFile("_"+geomName, extname);

                boolean createIndex = !indexfile.exists();

                GeometryManager geomManager = GeometryLocator.getGeometryManager();

                SpatialIndexFactory indexfactory = geomManager.getSpatialIndexFactory(indexTypeName);

                DynObject params = indexfactory.createParameters();

                params.setDynValue("file", indexfile);

                SpatialIndex index = geomManager.createSpatialIndex(indexTypeName, params);

                if (createIndex) {

                    I18nManager i18n = ToolsLocator.getI18nManager();

                    this.taskStatus.add();

                    taskStatus.message(i18n.getTranslation("_Creating_spatial_index"));

                    taskStatus.setRangeOfValues(0, this.virtualrows.size64());

                    taskStatus.setCurValue(0);

                    Envelope theEnvelope = geomManager.createEnvelope(Geometry.SUBTYPES.GEOM2D);

                    set = this.createSet(null, featureType);

                    it = set.fastIterator();

                    while (it.hasNext()) {

                        taskStatus.incrementCurrentValue();

                        if (taskStatus.isCancellationRequested()) {

                            taskStatus.cancel();

                            LOGGER.info("Spatial index creation cancelled (" + getFullFileName() + ")");

                            break;

                        }

                        FeatureProvider f = it.next();

                        if (f == null) {

                            continue;

                        }

                        Object oid = null;

                        try {

                            oid = f.getOID();

                            Geometry geom = (Geometry) f.get(geomdesc.getName());

                            if (geom != null) {

                                index.insert(geom, oid);

                                theEnvelope.add(geom);

                            }

                        } catch (Throwable ex) {

                            LOGGER.debug("Can't insert feature '" + Objects.toString(oid) + "' in spatial index.", ex);

                        }

                    }

                    taskStatus.message(i18n.getTranslation("_Saving_spatial_index"));

                    taskStatus.setIndeterminate();

                    index.flush();

                    if (!theEnvelope.isEmpty()) {

                        bboxFileSave("_"+geomName, theEnvelope);

                    }

                    this.envelopes.put(geomName,theEnvelope);

                } else {

                    this.envelopes.put(geomName,env);

                }

                this.spatialIndexes.put(geomdesc.getName(), index);

            }

            taskStatus.terminate();

        } catch (Exception ex) {

            taskStatus.abort();

            LOGGER.warn("Can't create spatial index.", ex);

        } finally {

            DisposeUtils.disposeQuietly(it);

            DisposeUtils.disposeQuietly(set);

            taskStatus.remove();

        }

    }

    public File getAuxFile(String extension) {

        return getAuxFile(null, extension);

    }

    public File getAuxFile(String suffix, String extension) {

        File data_file = SimpleReaderStoreParameters.getFile(this.getSimpleReaderParameters());

        if (data_file == null){

            return null;

        }

        File index_file;

        if(StringUtils.isBlank(suffix)){

            index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + "." + extension);

        } else {

            index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + suffix + "." + extension);

        }

        return index_file;

    }

    public SpatialIndex getSpatialIndex() {

        if(spatialIndexes == null){

            return null;

        }

        FeatureAttributeDescriptor geomdesc = this.featureType.getDefaultGeometryAttribute();

        if(geomdesc == null){

            return null;

        }

        return spatialIndexes.get(geomdesc.getName());

    }

    protected void bboxFileSave(Map<String,Envelope> envelopes) {

        for (Map.Entry<String, Envelope> entry : envelopes.entrySet()) {

            String key = entry.getKey();

            Envelope val = entry.getValue();

            bboxFileSave("_"+key, val);

        }

    }

    protected void bboxFileSave(Envelope envelope) {

        bboxFileSave((String)null, envelope);

    }

    protected void bboxFileSave(String suffix, Envelope envelope) {

        File bboxfile = this.getAuxFile(suffix,"bbox");

        bboxFileSave(bboxfile, envelope);

    }

    protected void bboxFileSave(File bboxfile, Envelope envelope) {

        if( envelope == null ) {

            bboxfile.delete();

            return;

        }

        try {

            FileUtils.write(

                    bboxfile, 

                    envelope.getBox2D().convertToWKTQuietly(), 

                    StandardCharsets.UTF_8

            );

        } catch(Exception ex) {

            LOGGER.warn("Can't write bbox file '"+Objects.toString(bboxfile)+"'.",ex);

        }

    }

    protected Envelope bboxFileLoad(String suffix) {

        File bboxfile = this.getAuxFile(suffix, "bbox");

        return bboxFileLoad(bboxfile);

    }

    protected Envelope bboxFileLoad(File bboxfile) {

        try {

            if( bboxfile.exists() ) {

                GeometryManager geomManager = GeometryLocator.getGeometryManager();

                String wkt = FileUtils.readFileToString(bboxfile, StandardCharsets.UTF_8);                    

                Geometry geom = geomManager.createFrom(wkt);

                if( geom!=null ) {

                    return geom.getEnvelope();

                }

            }

        } catch(Exception ex) {

            LOGGER.warn("Can't load bbox file",ex);

        }

        return null;

    }

    protected abstract SimpleReader getSimpleReader(SimpleReaderStoreParameters parameters, Reader in) throws IOException ;//{

//        return CSVUtils.getSimpleReader(getSimpleReaderParameters(), in);

//    }

}