Statistics
| Revision:

svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / csv / CSVStoreProvider.java @ 41084

History | View | Annotate | Download (32.6 KB)

1
/**
2
 * gvSIG. Desktop Geographic Information System.
3
 *
4
 * Copyright (C) 2007-2013 gvSIG Association.
5
 *
6
 * This program is free software; you can redistribute it and/or
7
 * modify it under the terms of the GNU General Public License
8
 * as published by the Free Software Foundation; either version 3
9
 * of the License, or (at your option) any later version.
10
 *
11
 * This program is distributed in the hope that it will be useful,
12
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
 * GNU General Public License for more details.
15
 *
16
 * You should have received a copy of the GNU General Public License
17
 * along with this program; if not, write to the Free Software
18
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
19
 * MA  02110-1301, USA.
20
 *
21
 * For any additional information, do not hesitate to contact us
22
 * at info AT gvsig.com, or visit our website www.gvsig.com.
23
 */
24
package org.gvsig.fmap.dal.store.csv;
25

    
26
import java.io.File;
27
import java.io.FileReader;
28
import java.io.FileWriter;
29
import java.io.IOException;
30
import java.net.URL;
31
import java.text.SimpleDateFormat;
32
import java.util.ArrayList;
33
import java.util.HashMap;
34
import java.util.Iterator;
35
import java.util.List;
36
import java.util.Locale;
37

    
38
import org.apache.commons.io.FilenameUtils;
39
import org.cresques.cts.IProjection;
40
import org.gvsig.fmap.dal.DALLocator;
41
import org.gvsig.fmap.dal.DataManager;
42
import org.gvsig.fmap.dal.DataServerExplorer;
43
import org.gvsig.fmap.dal.DataStore;
44
import org.gvsig.fmap.dal.DataStoreNotification;
45
import org.gvsig.fmap.dal.DataTypes;
46
import org.gvsig.fmap.dal.FileHelper;
47
import org.gvsig.fmap.dal.exception.DataException;
48
import org.gvsig.fmap.dal.exception.InitializeException;
49
import org.gvsig.fmap.dal.exception.OpenException;
50
import org.gvsig.fmap.dal.exception.ReadException;
51
import org.gvsig.fmap.dal.exception.ValidateDataParametersException;
52
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;
53
import org.gvsig.fmap.dal.feature.EditableFeatureType;
54
import org.gvsig.fmap.dal.feature.Feature;
55
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;
56
import org.gvsig.fmap.dal.feature.FeatureSet;
57
import org.gvsig.fmap.dal.feature.FeatureStore;
58
import org.gvsig.fmap.dal.feature.FeatureType;
59
import org.gvsig.fmap.dal.feature.exception.PerformEditingException;
60
import org.gvsig.fmap.dal.feature.spi.FeatureProvider;
61
import org.gvsig.fmap.dal.feature.spi.FeatureStoreProviderServices;
62
import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider;
63
import org.gvsig.fmap.dal.resource.ResourceAction;
64
import org.gvsig.fmap.dal.resource.file.FileResource;
65
import org.gvsig.fmap.dal.resource.spi.ResourceConsumer;
66
import org.gvsig.fmap.dal.resource.spi.ResourceProvider;
67
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer;
68
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters;
69
import org.gvsig.fmap.dal.spi.DataStoreProviderServices;
70
import org.gvsig.fmap.geom.Geometry;
71
import org.gvsig.fmap.geom.GeometryLocator;
72
import org.gvsig.fmap.geom.GeometryManager;
73
import org.gvsig.fmap.geom.primitive.Envelope;
74
import org.gvsig.fmap.geom.primitive.Point;
75
import org.gvsig.fmap.geom.type.GeometryType;
76
import org.gvsig.tools.ToolsLocator;
77
import org.gvsig.tools.dataTypes.CoercionException;
78
import org.gvsig.tools.dataTypes.DataTypesManager;
79
import org.gvsig.tools.dataTypes.DataTypesManager.Coercion;
80
import org.gvsig.tools.dataTypes.DataTypesManager.CoercionWithLocale;
81
import org.gvsig.tools.dispose.DisposableIterator;
82
import org.gvsig.tools.dynobject.DynObject;
83
import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException;
84
import org.gvsig.tools.evaluator.AbstractEvaluator;
85
import org.gvsig.tools.evaluator.EvaluatorData;
86
import org.gvsig.tools.evaluator.EvaluatorException;
87
import org.gvsig.tools.exception.BaseException;
88
import org.gvsig.tools.exception.NotYetImplemented;
89
import org.gvsig.tools.persistence.PersistentState;
90
import org.gvsig.tools.persistence.exception.PersistenceException;
91
import org.gvsig.tools.task.SimpleTaskStatus;
92
import org.gvsig.tools.task.TaskStatusManager;
93
import org.gvsig.tools.visitor.VisitCanceledException;
94
import org.gvsig.tools.visitor.Visitor;
95
import org.slf4j.Logger;
96
import org.slf4j.LoggerFactory;
97
import org.supercsv.comment.CommentStartsWith;
98
import org.supercsv.io.CsvListReader;
99
import org.supercsv.io.CsvListWriter;
100
import org.supercsv.prefs.CsvPreference;
101
import org.supercsv.quote.QuoteMode;
102

    
103
public class CSVStoreProvider extends AbstractMemoryStoreProvider implements
104
ResourceConsumer {
105
    private static final Logger logger = LoggerFactory.getLogger(CSVStoreProvider.class);
106

    
107
    public static final String NAME = "CSV";
108
    public static final String DESCRIPTION = "CSV file";
109

    
110
    public static final String METADATA_DEFINITION_NAME = NAME;
111

    
112
    private ResourceProvider resource;
113

    
114
    private long counterNewsOIDs = 0;
115
    private Envelope envelope;
116
    private boolean need_calculate_envelope = false;
117
    private SimpleTaskStatus taskStatus;
118

    
119

    
120
    public CSVStoreProvider(CSVStoreParameters parameters,
121
        DataStoreProviderServices storeServices) throws InitializeException {
122
        super(
123
            parameters, 
124
            storeServices,
125
            FileHelper.newMetadataContainer(METADATA_DEFINITION_NAME)
126
        );
127

    
128
        TaskStatusManager manager = ToolsLocator.getTaskStatusManager();
129
        this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV");
130

    
131
        counterNewsOIDs = 0;
132

    
133
        File file = getCSVParameters().getFile();
134
        resource = this.createResource(
135
            FileResource.NAME,
136
            new Object[] { file.getAbsolutePath() }
137
        );
138

    
139
        resource.addConsumer(this);
140
        initializeFeatureTypes();
141
    }
142

    
143
    private CSVStoreParameters getCSVParameters() {
144
        return (CSVStoreParameters) this.getParameters();
145
    }
146

    
147
    public String getProviderName() {
148
        return NAME;
149
    }
150

    
151
    public boolean allowWrite() {
152
        return false;
153
    }
154

    
155
    private String getFullFileName() {
156
            // Usar solo para mostrar mensajes en el logger.
157
                String s = "(unknow)";
158
                try { 
159
                        s = getCSVParameters().getFile().getAbsolutePath();
160
                } catch(Exception e2) {
161
                        s = "(unknow)";
162
                }
163
                return s;
164
    }
165
    
166
    public void open() throws OpenException {
167
        if (this.data != null) {
168
            return;
169
        }
170
        this.data = new ArrayList<FeatureProvider>();
171
        resource.setData(new HashMap());
172
        counterNewsOIDs = 0;
173
                try {
174
                        loadFeatures();
175
                } catch (RuntimeException e) {
176
                        logger.warn("Can't load features from CSV '"+getFullFileName()+"'.", e);
177
                        throw e;
178
                } catch (Exception e) {
179
                        logger.warn("Can't load features from CSV '"+getFullFileName()+"'.", e);
180
                        throw new RuntimeException(e);
181
                }
182
    }
183

    
184
    public DataServerExplorer getExplorer() throws ReadException {
185
        DataManager manager = DALLocator.getDataManager();
186
        FilesystemServerExplorerParameters params;
187
        try {
188
            params = (FilesystemServerExplorerParameters) manager
189
            .createServerExplorerParameters(FilesystemServerExplorer.NAME);
190
            params.setRoot(this.getCSVParameters().getFile().getParent());
191
            return manager.openServerExplorer(FilesystemServerExplorer.NAME,params);
192
        } catch (DataException e) {
193
            throw new ReadException(this.getProviderName(), e);
194
        } catch (ValidateDataParametersException e) {
195
            throw new ReadException(this.getProviderName(), e);
196
        }
197

    
198
    }
199

    
200
    class Writer {
201
            private Envelope envelope = null;
202
                private boolean calculate_envelope = false;
203
                private CsvListWriter listWriter = null;
204
                private CsvPreference csvpreferences = null;
205
                private FileWriter fwriter = null;
206
                private FeatureType ftype;
207
                private File file;
208
                private String[] values;
209
                private FeatureAttributeDescriptor[] descriptors;
210
                private Coercion convert = null;
211
                private int errorcounts =0;
212
                private Throwable lasterror = null;
213
                private Locale locale = null;
214
            
215
            public void initialize(File file, FeatureType ftype, CsvPreference csvpreferences) {
216
                    this.file = file;
217
                    this.ftype = ftype;
218
                        this.csvpreferences = csvpreferences;
219
                        this.locale = CSVStoreParameters.getLocale(getCSVParameters());
220
                    if( csvpreferences == null ) {
221
                            this.csvpreferences = CsvPreference.STANDARD_PREFERENCE;
222
                    }
223
                    if( ftype.getDefaultGeometryAttributeName() != null ) {
224
                            this.calculate_envelope = true;
225
                    }
226
                    this.descriptors = this.ftype.getAttributeDescriptors();
227
                    this.convert = ToolsLocator.getDataTypesManager().getCoercion(org.gvsig.tools.dataTypes.DataTypes.STRING);
228
                    this.errorcounts = 0;
229
            }
230
            
231
            public void begin() {
232
                    try {
233
                                this.fwriter = new FileWriter(file);
234
                        } catch (IOException e) {
235
                                logger.warn("Can't open file for write ("+file.getAbsolutePath()+").",e);
236
                                throw new RuntimeException(e);
237
                        }
238
                    this.listWriter = new CsvListWriter(this.fwriter,this.csvpreferences);
239
                    int n = 0;
240
                    for(int i=0; i<descriptors.length; i++ ) {
241
                            FeatureAttributeDescriptor descriptor = descriptors[i];
242
                            if( descriptor.getEvaluator()== null ) {
243
                                    n++;
244
                            }
245
                    }
246
                            
247
                    String[] header = new String[n];
248
                    this.values = new String[n];
249
                    n = 0;
250
                    for(int i=0; i<descriptors.length; i++ ) {
251
                            FeatureAttributeDescriptor descriptor = descriptors[i];
252
                            if( descriptor.getEvaluator()== null ) {
253
                                    String name = descriptor.getName();
254
                                    String typeName = descriptor.getDataTypeName();
255
                                    if( descriptor.getDataType().getType() == DataTypes.STRING ) {
256
                                            header[n++] = name + "__" + typeName + "__" + descriptor.getSize();
257
                                    } else {
258
                                            header[n++] = name + "__" + typeName;
259
                                    }
260
                            }
261
                    }
262
            try {
263
                                listWriter.writeHeader(header);
264
                        } catch (Exception e) {
265
                                logger.warn("Can't write header '"+header.toString()+"' file for write ("+file.getAbsolutePath()+").",e);
266
                                throw new RuntimeException(e);
267
                        }
268
            }
269
            
270
            public void add(FeatureProvider feature) {
271
                        if (this.calculate_envelope) {
272
                                Geometry geom = feature.getDefaultGeometry();
273
                                if (geom != null) {
274
                                        if (envelope == null) {
275
                                                try {
276
                                                        envelope = (Envelope) geom.getEnvelope().clone();
277
                                                } catch (CloneNotSupportedException e) {
278
                                                        logger.warn("Este error no deberia pasar, siempre se puede hacer un clone de un envelope.",e);
279
                                                }
280
                                        } else {
281
                                                envelope.add(geom.getEnvelope());
282
                                        }
283
                                }
284
                        }
285
                    int n = 0;
286
                    for(int i=0; i<descriptors.length; i++ ) {
287
                            FeatureAttributeDescriptor descriptor = descriptors[i];
288
                            if( descriptor.getEvaluator()== null ) {
289
                                        Object value = feature.get(i);
290
                                    try {
291
                                            n++;
292
                                            if( this.convert!=null && this.convert instanceof CoercionWithLocale ) {
293
                                                    values[n] = (String) ((CoercionWithLocale)this.convert).coerce(value,this.locale);
294
                                            } else {
295
                                                    values[n] = (String) this.convert.coerce(value);
296
                                            }
297
                                        } catch (CoercionException e) {
298
                                                try {
299
                                                        values[n] = value.toString();
300
                                                } catch(Exception ex) {
301
                                                        values[n] = "";
302
                                                }
303
                                                if( errorcounts++ <= 10 ) {
304
                                                        this.lasterror = e;
305
                                                        logger.warn("Can't convert value of field "+i+" to string in CVS file '"+getFullFileName()+"'.",e);
306
                                                        if( errorcounts == 10 ) {
307
                                                                logger.warn("Too many error writing CVS file '"+getFullFileName()+"', don't output more.");
308
                                                        }
309
                                                } 
310
                                        } 
311
                            }
312
                    }
313
                    try {
314
                                this.listWriter.writeHeader(values);
315
                        } catch (IOException e) {
316
                                if( errorcounts++ <= 10 ) {
317
                                        this.lasterror = e;
318
                                        logger.warn("Can't write values to CVS file '"+getFullFileName()+"'.",e);
319
                                        if( errorcounts == 10 ) {
320
                                                logger.warn("Too many error writing CVS file '"+getFullFileName()+"', don't output more.");
321
                                        }
322
                                } 
323
                        }
324
                        
325
            }
326
            
327
            public void end() throws PerformEditingException {
328
                    if( this.errorcounts>0 ) {
329
                            throw new PerformEditingException(this.file.getAbsolutePath(), lasterror);
330
                    }
331
                        if( listWriter!=null ) {
332
                            try {
333
                                    listWriter.close();
334
                            } catch(Exception ex) {
335
                                    // Ignore error
336
                            }
337
                                listWriter=null ;
338
                        }
339
                        if( fwriter!=null ) {
340
                            try {
341
                                    fwriter.close();
342
                            } catch(Exception ex) {
343
                                    // Ignore error
344
                            }
345
                                fwriter=null ;
346
                        }
347
            }
348
            
349
            public Envelope getEnvelope() {
350
                    return this.envelope;
351
            }
352
    }
353

    
354
    public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException {
355

    
356
        try {
357
            this.taskStatus.add();
358
            taskStatus.message("_preparing");
359
            getResource().execute(new ResourceAction() {
360
                public Object run() throws Exception {
361
                    FeatureSet features = null;
362
                    DisposableIterator it = null;
363
                    try {
364
                        File file = (File) resource.get();
365
                        
366
                        Writer writer = new Writer();
367
                        writer.initialize(file,getStoreServices().getDefaultFeatureType(),getCSVPreferences());
368
                        features =
369
                            getStoreServices().getFeatureStore()
370
                            .getFeatureSet();
371
                        List<FeatureProvider> newdata = new ArrayList<FeatureProvider>();
372
                        writer.begin();
373
                        it = features.fastIterator();
374
                        taskStatus.setRangeOfValues(0,0);
375
                        long counter=0;
376
                        while (it.hasNext()) {
377
                            taskStatus.setCurValue(counter++);
378
                            FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature(
379
                                (org.gvsig.fmap.dal.feature.Feature) it.next());
380
                            writer.add(feature);
381
                            if (feature.getOID() == null){
382
                                logger.warn("feature without OID");
383
                                feature.setOID(createNewOID());
384
                            }
385
                            newdata.add(feature);
386
                        }
387
                        data = newdata;
388
                        if (writer.getEnvelope() != null){
389
                            envelope = writer.getEnvelope().getGeometry().getEnvelope();
390
                        }
391
                        resource.notifyChanges();
392
                        writer.end();
393
                    } finally {
394
                        if (it != null) {
395
                            it.dispose();
396
                        }
397
                        if (features != null) {
398
                            features.dispose();
399
                        }
400
                    }
401
                    return null;
402
                }
403
            });
404
            this.taskStatus.terminate();
405
        } catch (Exception e) {
406
            this.taskStatus.abort();
407
            throw new PerformEditingException(getResource().toString(), e);
408
        } finally {
409
            this.taskStatus.remove();
410
        }
411
    }
412

    
413
    public boolean closeResourceRequested(ResourceProvider resource) {
414
        return true;
415
    }
416

    
417
    public int getOIDType() {
418
        return DataTypes.LONG;
419
    }
420

    
421
    public boolean supportsAppendMode() {
422
        return false;
423
    }
424

    
425
    public void append(FeatureProvider featureProvider) {
426
            throw new UnsupportedOperationException();
427
    }
428

    
429
    public void beginAppend() {
430
            throw new UnsupportedOperationException();
431
    }
432

    
433
    public void endAppend() {
434
            throw new UnsupportedOperationException();
435
    }
436

    
437
    public void saveToState(PersistentState state) throws PersistenceException {
438
        throw new NotYetImplemented();
439
    }
440

    
441
    public void loadFromState(PersistentState state) throws PersistenceException {
442
        throw new NotYetImplemented();
443
    }
444

    
445
    public Object createNewOID() {
446
        return new Long(counterNewsOIDs++);
447
    }
448

    
449
    protected void initializeFeatureTypes() throws InitializeException {
450
        try {
451
            this.open();
452
        } catch (OpenException e) {
453
            throw new InitializeException(this.getProviderName(), e);
454
        }
455
    }
456

    
457
    public Envelope getEnvelope() throws DataException {
458
        this.open();
459
        if( this.envelope!= null )  {
460
                return this.envelope;
461
        }
462
        if( !this.need_calculate_envelope ) {
463
                return null;
464
        }
465
        FeatureStore fs = this.getFeatureStore();
466
        FeatureType ft = fs.getDefaultFeatureType();
467
        FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex());
468

    
469
        try {
470
            this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());
471
                        fs.accept(new Visitor() {
472
                                public void visit(Object obj) throws VisitCanceledException, BaseException {
473
                                        Feature f = (Feature) obj;
474
                                        Geometry geom = f.getDefaultGeometry();
475
                                        envelope.add(geom.getEnvelope());
476
                                }
477
                        });
478
                } catch (BaseException e) {
479
                        logger.warn("Can't calculate the envelope of CSV file '"+this.getFullName()+"'.",e);
480
                        this.envelope = null;
481
                }
482
        
483
        this.need_calculate_envelope = false;
484
        return this.envelope;
485
    }
486

    
487
    public Object getDynValue(String name) throws DynFieldNotFoundException {
488
        if( DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name) ) {
489
            try {
490
                return this.getEnvelope();
491
            } catch (DataException e) {
492
                return null;
493
            }
494
        } else {
495
            if( DataStore.METADATA_CRS.equalsIgnoreCase(name) ) {
496
                IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters());
497
                if (pro != null){
498
                    return pro;
499
                }
500
            }
501
        }
502
        return super.getDynValue(name);
503
    }
504

    
505

    
506
    public void resourceChanged(ResourceProvider resource) {
507
        this.getStoreServices().notifyChange(
508
            DataStoreNotification.RESOURCE_CHANGED,
509
            resource);
510
    }
511

    
512

    
513
    public Object getSourceId() {
514
        return this.getCSVParameters().getFile();
515
    }
516

    
517
    public String getName() {
518
        String name = this.getCSVParameters().getFile().getName();
519
        return FilenameUtils.getBaseName(name);
520
    }
521

    
522
    public String getFullName() {
523
        return this.getCSVParameters().getFile().getAbsolutePath();
524
    }
525

    
526
    public ResourceProvider getResource() {
527
        return resource;
528
    }
529

    
530
    private boolean isEmpty(String s) {
531
            if( s==null ) {
532
                    return true;
533
            }
534
            return s.trim().length()==0;
535
    }
536
    
537
        private CsvPreference getCSVPreferences() {
538
                try {
539
                        String s = null;
540
                        char quoteChar;
541
                        int delimiterChar;
542
                        String endOfLineSymbols;
543

    
544
                        DynObject params = this.getParameters();
545

    
546
                        CsvPreference.Builder builder = null;
547

    
548
                        CsvPreference defaultPreference = CSVStoreParameters
549
                                        .getPredefinedCSVPreferences(params);
550
                        if (defaultPreference == null) {
551
                                defaultPreference = CsvPreference.STANDARD_PREFERENCE;
552
                        }
553

    
554
                        endOfLineSymbols = CSVStoreParameters.getRecordSeparator(params);
555
                        if (isEmpty(endOfLineSymbols)) {
556
                                endOfLineSymbols = defaultPreference.getEndOfLineSymbols();
557
                        }
558
                        s = CSVStoreParameters.getQuoteCharacter(params);
559
                        if (isEmpty(s)) {
560
                                quoteChar = (char) defaultPreference.getQuoteChar();
561
                        } else {
562
                                quoteChar = s.charAt(0);
563
                        }
564
                        s = CSVStoreParameters.getDelimiter(params);
565
                        if (isEmpty(s)) {
566
                                delimiterChar = defaultPreference.getDelimiterChar();
567
                        } else {
568
                                delimiterChar = s.charAt(0);
569
                        }
570

    
571
                        builder = new CsvPreference.Builder(quoteChar, delimiterChar,
572
                                        endOfLineSymbols);
573

    
574
                        s = CSVStoreParameters.getCommentStartMarker(params);
575
                        if (!isEmpty(s)) {
576
                                CommentStartsWith cs = new CommentStartsWith(s);
577
                                builder.skipComments(cs);
578
                        }
579

    
580
                        builder.surroundingSpacesNeedQuotes(CSVStoreParameters
581
                                        .getSurroundingSpacesNeedQuotes(params));
582
                        QuoteMode quoteMode = CSVStoreParameters.getQuoteMode(params);
583
                        if( quoteMode != null ) {
584
                                builder.useQuoteMode(quoteMode);
585
                        }
586
                        return builder.build();
587
                } catch (Exception e) {
588
                        logger.warn("Can't make preferences for CSV '" + getFullFileName()
589
                                        + "'.", e);
590
                        return null;
591
                }
592
        }
593
    
594
    private EditableFeatureType getFeatureType(String headers[], int automaticTypes[]) {
595
                EditableFeatureType fType =        getStoreServices().createFeatureType(this.getName());
596
                fType.setHasOID(true);
597
                DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager();
598
                
599
            int[] types = new int[headers.length];
600
            int[] sizes = new int[headers.length];
601
            //
602
            // Calculamos cuales pueden ser los tipos de datos
603
            //
604
            
605
            // Por defecto todos string
606
            for( int i=0; i<types.length ; i++) {
607
                    types[i] = DataTypes.STRING;
608
                    sizes[i] = 0;
609
            }
610
            
611
            // Luego asuminos los tipos pasados por parametro, que se supone
612
            // son los detectados automaticamente.
613
                if (automaticTypes != null) {
614
                        for (int i = 0; i < types.length && i < automaticTypes.length; i++) {
615
                                types[i] = automaticTypes[i];
616
                        }
617
                }
618
            
619
            // Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo
620
            // los tipos anteriores en caso de definirse en la cabezara.
621
                // El formato de la cabecera seria:
622
                //   name[:typename[:size]]
623
                //   name[__typename[__size]]
624
                //
625
            for( int i=0; i<types.length; i++) {
626
                    String s = headers[i];
627
                    String typename = null;
628
                    String[] ss = null;
629
                    if( s.contains(":") ) {
630
                            ss = s.split(":");
631
                    } else if( s.contains("__") ) {
632
                        ss = s.split("__");
633
                    }
634
                    if( ss!=null ) {
635
                            headers[i] = ss[0];
636
                            typename = ss[1];
637
                            types[i] = dataTypesManager.getType(typename); 
638
                            if( types[i]==DataTypes.INVALID ) {
639
                                    types[i] = DataTypes.STRING;
640
                                    logger.info("Type '"+typename+"' not valid for attribute '"+s+"' in CSV file '"+this.getFullFileName()+"'.");
641
                            }
642
                            if( types[i]== DataTypes.STRING && ss.length>2 ) {
643
                                    try {
644
                                            sizes[i] = Integer.parseInt(ss[2]);
645
                                    } catch(Exception ex) {
646
                                            logger.warn("Ignore incorrect field size for field "+i+" ("+s+") in CSV header of '"+getFullFileName()+"'.",ex);
647
                                    }
648
                            }
649
                    }
650
            }
651
            
652
            // Y por ultimo hacemos caso a lo que se haya especificado en los parametros
653
            // de apertura del CSV, teniendo esto prioridad sobre todo.
654
            int[] param_types = CSVStoreParameters.getFieldTypes(this.getParameters());
655
            if( param_types != null ) {
656
                for( int i=0; i<types.length && i<param_types.length; i++) {
657
                        types[i] = param_types[i];
658
                }
659
            }
660
            
661
            //
662
            // Una vez ya sabemos los tipos de datos rellenamos el feature-type
663
            //
664
            int[] param_sizes = CSVStoreParameters.getFieldSizes(this.getParameters());
665
            for(int i=0; i<types.length; i++) {
666
                        int fieldType = types[i];
667
                        String fieldName = headers[i];
668
                        EditableFeatureAttributeDescriptor fad = fType.add(fieldName, fieldType);
669
                        if( param_sizes!=null && i<param_sizes.length && param_sizes[i]>0 ) {
670
                                fad.setSize(param_sizes[i]);
671
                        } else {
672
                                fad.setSize(sizes[i]);
673
                        }
674
                        if( fieldType == DataTypes.GEOMETRY && fType.getDefaultGeometryAttributeName() == null) {
675
                                fType.setDefaultGeometryAttributeName(fieldName);
676
                        }
677
                }
678
            String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters());
679
            if( pointDimensionNames!= null ) {
680
                    EditableFeatureAttributeDescriptor attr = fType.add("GEOM", DataTypes.GEOMETRY, new ToPointEvaluaror(pointDimensionNames));
681
                    GeometryManager geommgr = GeometryLocator.getGeometryManager();
682
                    GeometryType gt;
683
                        try {
684
                                gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D);
685
                            attr.setGeometryType(gt);
686
                        } catch (Exception e) {
687
                                logger.warn("Can't set geometry type for the calculated field in CSV file '"+getFullFileName()+"'.",e);
688
                        }
689
            }
690
                return fType;
691
    }
692
    
693
    static class ToPointEvaluaror extends AbstractEvaluator {
694

    
695
            private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class);
696
            
697
            private GeometryManager geommgr = null;
698
            private String xname = null;
699
            private String yname = null;
700
            private String zname = null;
701
                private Coercion toDouble;
702
                private int errorcount = 0;
703
            ToPointEvaluaror(String[] pointDimensionNames) {
704
                    this.xname = pointDimensionNames[0];
705
                    this.yname = pointDimensionNames[1];
706
                    if( pointDimensionNames.length >2 ) {
707
                            this.yname = pointDimensionNames[2];
708
                    }
709
                    this.geommgr = GeometryLocator.getGeometryManager();
710
                    this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);
711
            }
712
            
713
                public Object evaluate(EvaluatorData data) throws EvaluatorException {
714
                        try {
715
                                double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue();
716
                                double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue();
717
                                Point point = geommgr.createPoint(x, y,Geometry.SUBTYPES.GEOM3D);
718
                                if (zname != null) {
719
                                        double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue();
720
                                        point.setCoordinateAt(2, z);
721
                                }
722
                                return point;
723
                        } catch (Exception ex) {
724
                                if( ++errorcount <5 ) {
725
                                        logger.warn("["+errorcount+"] Can't create point in CSV provider. XNAME='"+
726
                                                        xname + "', YNAME='"+yname+"', ZNAME='"+zname+"', data="+data.toString());
727
                                }
728
                                return null;
729
                        }
730
                }
731

    
732
                public String getName() {
733
                        return "ToPointEvaluaror";
734
                }
735
            
736
    }
737
    
738
        private void loadFeatures() throws IOException, DataException,
739
                        CoercionException, CloneNotSupportedException {
740
                //
741
                // http://supercsv.sourceforge.net/examples_reading.html
742
                // http://supercsv.sourceforge.net/apidocs/index.html
743
                //
744
                FileReader in = null;
745
                CsvListReader reader = null;
746
                try {
747
                        String headers[] = null;
748
                        FeatureStoreProviderServices store = this.getStoreServices();
749

    
750
                        boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
751
                        
752
                        // Initiaize the CSV parser
753
                        CsvPreference preferences = getCSVPreferences();
754
                        in = new FileReader(this.getCSVParameters().getFile());
755

    
756
                        reader = new CsvListReader(in, preferences);
757
                        headers = CSVStoreParameters.getHeaders(getCSVParameters());
758
                        if (headers == null) {
759
                                headers = reader.getHeader(true);
760
                                if (headers == null) {
761
                                        String msg = "Can't retrieve header from csv file '"
762
                                                        + this.getCSVParameters().getFile()
763
                                                                        .getAbsolutePath()
764
                                                        + "' and not specified in the parameters.";
765
                                        logger.warn(msg);
766
                                        throw new RuntimeException(msg);
767
                                }
768
                        }
769

    
770
                        // Initialize the feature types
771
                        EditableFeatureType edftype = this.getFeatureType(headers, automaticDetectionOfTypes());
772
                        FeatureType ftype = edftype.getNotEditableCopy();
773
                        List<FeatureType> ftypes = new ArrayList<FeatureType>();
774
                        ftypes.add(ftype);
775
                        store.setFeatureTypes(ftypes, ftype);
776

    
777
                        Coercion coercion[] = new Coercion[ftype.size()];
778
                        int sizes[] = new int[ftype.size()];
779
                        for (int i = 0; i < ftype.size(); i++) {
780
                                sizes[i] = -1;
781
                                FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i);
782
                                coercion[i] = ad.getDataType().getCoercion();
783
                                if( ad.getDataType().getType() == DataTypes.STRING ) {
784
                                        if( ad.getSize() == 0 ) {
785
                                                // Es un string y no tiene un size asignado.
786
                                                // Lo ponemos a cero para calcularlo.
787
                                                sizes[i] = 0;
788
                                        }
789
                                }
790
                        }
791
                        if( ftype.getDefaultGeometryAttributeName() != null) {
792
                                this.need_calculate_envelope = true;
793
                        }
794
                        
795
                        Locale locale = CSVStoreParameters.getLocale(getCSVParameters());
796
                        taskStatus.message("_loading");
797
                        int count = 0;
798

    
799
                        int count_errors = 0;
800
                        List<String> row = reader.read();
801
                        
802
                        while (row != null) {
803
                                taskStatus.setCurValue(++count);
804
                                FeatureProvider feature = this.createFeatureProvider(ftype);
805
                                for (int i = 0; i < row.size(); i++) {
806
                                        Object rawvalue = row.get(i);
807
                                        try {
808
                                                Object value = null;
809
                                                if( locale != null && coercion[i] instanceof CoercionWithLocale ) {
810
                                                        value = ((CoercionWithLocale)(coercion[i])).coerce(rawvalue, locale);
811
                                                } else {
812
                                                        value = coercion[i].coerce(rawvalue);
813
                                                }
814
                                                feature.set(i, value);
815
                                                if (sizes[i] >= 0 && value != null) {
816
                                                        int x = ((String) value).length();
817
                                                        if (sizes[i] < x) {
818
                                                                sizes[i] = x;
819
                                                        }
820
                                                }
821
                                        } catch (RuntimeException ex) {
822
                                                if (!ignore_errors) {
823
                                                        throw ex;
824
                                                }
825
                                                if( count_errors++ <10 ) {
826
                                                        logger.warn("Can't load value of attribute "+i+" in row "+count+".",ex);
827
                                                }
828
                                                if( count_errors==10 ) {
829
                                                        logger.info("Too many errors, suppress messages.");
830
                                                }
831
                                        }
832
                                }
833
                                this.addFeatureProvider(feature);
834
                                row = reader.read();
835
                        }
836
                        for (int i = 0; i < ftype.size(); i++) {
837
                                if( sizes[i]>0 ) {
838
                                        EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor)edftype.getAttributeDescriptor(i));
839
                                        efad.setSize(sizes[i]);
840
                                }
841
                        }
842
                        // Volvemos a asignar al store el featuretype, ya que puede
843
                        // haber cambiado.
844
                        ftype = edftype.getNotEditableCopy();
845
                        ftypes = new ArrayList<FeatureType>();
846
                        ftypes.add(ftype);
847
                        store.setFeatureTypes(ftypes, ftype);
848

    
849
                        taskStatus.terminate();
850
                } finally {
851
                        if( reader != null ) {
852
                                try { 
853
                                        reader.close();
854
                                } catch(Exception ex) {
855
                                        // Do nothing
856
                                }
857
                                reader = null;
858
                        }
859
                        if( in != null ) {
860
                                try { 
861
                                        in.close();
862
                                } catch(Exception ex) {
863
                                        // Do nothing
864
                                }
865
                                in = null;
866
                        }
867
                }
868
        }
869
        
870
        private int[] automaticDetectionOfTypes() throws IOException {
871
                boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters());
872
                if( !automatic_types_detection ) {
873
                        return  null;
874
                }
875
                
876
                final int T_INT = 0;
877
                final int T_FLOAT = 1;
878
                final int T_DOUBLE = 2;
879
                final int T_LONG = 3;
880
                final int T_URL = 4;
881
                final int T_DATE = 5;
882
                boolean possibleDataTypes[][] = null;
883
                int[] types = null;
884

    
885
                FileReader in = null;
886
                CsvListReader reader = null;
887
                String headers[] = null;
888
                SimpleDateFormat dateFormat = new SimpleDateFormat();
889

    
890
                try {
891
                        CsvPreference preferences = getCSVPreferences();
892
                        in = new FileReader(this.getCSVParameters().getFile());
893

    
894
                        reader = new CsvListReader(in, preferences);
895
                        headers = reader.getHeader(true);
896
                        if (headers == null) {
897
                                headers = CSVStoreParameters.getHeaders(getCSVParameters());
898
                        }
899
                        types = new int[headers.length]; 
900

    
901
                        possibleDataTypes = new boolean[headers.length][6]; 
902
                        for (int i = 0; i < possibleDataTypes.length; i++) {
903
                                for( int j=0; j<4; j++ ) {
904
                                        possibleDataTypes[i][j] = true;  
905
                                }
906
                        }
907
                        List<String> row = reader.read();
908

    
909
                        while (row != null) {
910
                                for (int i = 0; i < row.size(); i++) {
911
                                        Object rawvalue = row.get(i);
912
                                        Object value = null;
913
                                        if( possibleDataTypes[i][T_DOUBLE] ) {
914
                                                try {
915
                                                        value = Double.parseDouble((String) rawvalue);
916
                                                        possibleDataTypes[i][T_DOUBLE] = true;
917
                                                } catch (Exception ex) {
918
                                                        possibleDataTypes[i][T_DOUBLE] = false;
919
                                                }
920
                                        }
921
                                        if( possibleDataTypes[i][T_FLOAT] ) {
922
                                                try {
923
                                                        value = Float.parseFloat((String) rawvalue);
924
                                                        possibleDataTypes[i][T_FLOAT] = true;
925
                                                } catch (Exception ex) {
926
                                                        possibleDataTypes[i][T_FLOAT] = false;
927
                                                }
928
                                        }
929
                                        if (possibleDataTypes[i][T_LONG]) {
930
                                                try {
931
                                                        value = Long.parseLong((String) rawvalue);
932
                                                        possibleDataTypes[i][T_LONG] = true;
933
                                                } catch (Exception ex) {
934
                                                        possibleDataTypes[i][T_LONG] = false;
935
                                                }
936
                                        }
937
                                        if (possibleDataTypes[i][T_INT]) {
938
                                                try {
939
                                                        value = Integer.parseInt((String) rawvalue);
940
                                                        possibleDataTypes[i][T_INT] = true;
941
                                                } catch (Exception ex) {
942
                                                        possibleDataTypes[i][T_INT] = false;
943
                                                }
944
                                        }                                        
945
                                        if (possibleDataTypes[i][T_DATE]) {
946
                                                try {
947
                                                        value = dateFormat.parse((String) rawvalue);
948
                                                        possibleDataTypes[i][T_DATE] = true;
949
                                                } catch (Exception ex) {
950
                                                        possibleDataTypes[i][T_DATE] = false;
951
                                                }
952
                                        }                                        
953
                                        if (possibleDataTypes[i][T_URL]) {
954
                                                try {
955
                                                        value = new URL((String) rawvalue);
956
                                                        possibleDataTypes[i][T_URL] = true;
957
                                                } catch (Exception ex) {
958
                                                        possibleDataTypes[i][T_URL] = false;
959
                                                }
960
                                        }                                        
961
                                }
962
                                row = reader.read();
963
                        }
964
                } finally {
965
                        if (reader != null) {
966
                                try {
967
                                        reader.close();
968
                                } catch (Exception ex) {
969
                                        // Do nothing
970
                                }
971
                                reader = null;
972
                        }
973
                        if (in != null) {
974
                                try {
975
                                        in.close();
976
                                } catch (Exception ex) {
977
                                        // Do nothing
978
                                }
979
                                in = null;
980
                        }
981

    
982
                }
983
                for (int i = 0; i < possibleDataTypes.length ; i++) {
984
                        if( possibleDataTypes[i][T_INT] ) {
985
                                types[i] = DataTypes.INT;
986
                                continue;
987
                        }
988
                        if( possibleDataTypes[i][T_LONG] ) {
989
                                types[i] = DataTypes.LONG;
990
                                continue;
991
                        }
992
                        if( possibleDataTypes[i][T_FLOAT] ) {
993
                                types[i] = DataTypes.FLOAT;
994
                                continue;
995
                        }
996
                        if( possibleDataTypes[i][T_DOUBLE] ) {
997
                                types[i] = DataTypes.DOUBLE;
998
                                continue;
999
                        }
1000
                        if( possibleDataTypes[i][T_URL] ) {
1001
                                types[i] = DataTypes.URL;
1002
                                continue;
1003
                        }
1004
                        if( possibleDataTypes[i][T_DATE] ) {
1005
                                types[i] = DataTypes.DATE;
1006
                                continue;
1007
                        }
1008
                        types[i] = DataTypes.STRING;
1009
                }
1010
                return types;
1011
        }
1012

    
1013
}