Revision 44251 trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.lib/src/main/java/org/gvsig/fmap/dal/feature/spi/simpleprovider/SimpleSequentialReaderStoreProvider.java

View differences:

SimpleSequentialReaderStoreProvider.java
26 26
import java.io.File;
27 27
import java.io.IOException;
28 28
import java.util.ArrayList;
29
import java.util.Collections;
29 30
import java.util.HashMap;
30 31
import java.util.Iterator;
31 32
import java.util.List;
32 33
import java.util.Locale;
34
import java.util.Map;
35
import org.apache.commons.io.FilenameUtils;
33 36

  
34 37
import org.apache.commons.lang3.StringUtils;
35 38
import org.cresques.cts.IProjection;
......
62 65
import org.gvsig.fmap.dal.resource.spi.ResourceProvider;
63 66
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer;
64 67
import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters;
68
import org.gvsig.fmap.dal.spi.DALSPILocator;
69
import org.gvsig.fmap.dal.spi.DataManagerProviderServices;
65 70
import org.gvsig.fmap.dal.spi.DataStoreProviderServices;
66 71
import org.gvsig.fmap.geom.Geometry;
67 72
import org.gvsig.fmap.geom.GeometryLocator;
......
71 76
import org.gvsig.fmap.geom.primitive.Envelope;
72 77
import org.gvsig.fmap.geom.primitive.Point;
73 78
import org.gvsig.fmap.geom.type.GeometryType;
79
import org.gvsig.timesupport.Interval;
74 80
import org.gvsig.tools.ToolsLocator;
75 81
import org.gvsig.tools.dataTypes.CoercionException;
76 82
import org.gvsig.tools.dataTypes.DataTypesManager;
77 83
import org.gvsig.tools.dataTypes.DataTypesManager.Coercion;
78 84
import org.gvsig.tools.dataTypes.DataTypesManager.CoercionWithLocale;
85
import org.gvsig.tools.dynobject.Tags;
79 86
import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException;
80 87
import org.gvsig.tools.exception.BaseException;
81 88
import org.gvsig.tools.task.SimpleTaskStatus;
......
91 98

  
92 99
    private static final Logger LOGGER = LoggerFactory.getLogger(SimpleSequentialReaderStoreProvider.class);
93 100

  
101
    private class ReaderData {
102
        private FeatureType defaultFeatureType;
103
        private List<FeatureType> featureTypes;
104
        private List<FeatureProvider> features;
105
        private boolean needCalculateEnvelope;
106
        private String name;
107
        private long OIDcounter;
108
        private DataStore store;
94 109

  
110
        public ReaderData() {
111
            this.needCalculateEnvelope = false;
112
            this.features = new ArrayList<>();
113
            this.OIDcounter = 0;
114
            this.store = null;
115
        }
116
        
117
        public void addFeatureProvider(FeatureProvider feature) {
118
		feature.setOID(this.OIDcounter++);
119
		this.features.add(feature);
120
        }
121
        
122
        public void setFeatureTypes(List<FeatureType> featureTypes, FeatureType defaultFeatureType) {
123
            this.featureTypes = featureTypes;
124
            this.defaultFeatureType = defaultFeatureType;
125
        }
126
        
127
        public void setNeedCalculateEnvelope(boolean needCalculateEnvelope) {
128
            this.needCalculateEnvelope = needCalculateEnvelope;
129
        }
130
        
131
        public boolean getNeedCalculateEnvelope() {
132
            return this.needCalculateEnvelope;
133
        }
134

  
135
        public String getName() {
136
            return name;
137
        }
138

  
139
        public void setName(String name) {
140
            this.name = name;
141
        }
142

  
143
        public List<FeatureType> getFeatureTypes() {
144
            return this.featureTypes;
145
        }
146

  
147
        public FeatureType getDefaultFeatureType() {
148
            return this.defaultFeatureType;
149
        }
150
        
151
        public List<FeatureProvider> getFeatures() {
152
            return this.features;
153
        }
154
        
155
        public DataStore getStore() throws InitializeException {
156
            if( this.store == null ) {
157
                SimpleSequentialReaderStoreProvider provider = new SimpleSequentialReaderStoreProvider(
158
                        readerFactory,
159
                        getParameters(),
160
                        null,
161
                        childrenData,
162
                        this
163
                );
164
                DataManagerProviderServices manager = DALSPILocator.getDataManagerProviderServices();
165
                this.store = manager.openStore(
166
                        getParameters(), 
167
                        provider
168
                );
169
                provider.setStoreServices((FeatureStoreProviderServices) this.store);
170
                provider.name = this.name;
171
            }
172
            return this.store;
173
        }
174
    }
175

  
95 176
    private final ResourceProvider resource;
96 177

  
97
    private long counterNewsOIDs = 0;
98 178
    private Envelope envelope;
99 179
    private boolean need_calculate_envelope = false;
100 180
    private final SimpleTaskStatus taskStatus;
101 181
    private String name = "";
102 182
    private final SimpleSequentialReaderFactory readerFactory;
183
    private final List<ReaderData> childrenData;
184
    private ReaderData readerData;
103 185

  
104 186
    public SimpleSequentialReaderStoreProvider(
105 187
            SimpleSequentialReaderFactory readerFactory,
106 188
            SimpleSequentialReaderStoreParameters parameters,
107 189
            DataStoreProviderServices storeServices
108 190
        ) throws InitializeException {
191
        this(readerFactory, parameters, storeServices, new ArrayList<ReaderData>(), null);
192
    }
193
    
194
    private SimpleSequentialReaderStoreProvider(
195
            SimpleSequentialReaderFactory readerFactory,
196
            SimpleSequentialReaderStoreParameters parameters,
197
            DataStoreProviderServices storeServices,
198
            List<ReaderData> childrenData,
199
            ReaderData readerData
200
            
201
        ) throws InitializeException {
109 202
        super(
110 203
                parameters,
111 204
                storeServices,
112 205
                FileHelper.newMetadataContainer(readerFactory.getName())
113 206
        );
207
        this.childrenData = childrenData;
208
        this.readerData = readerData;
114 209
        this.readerFactory = readerFactory;
115 210
        TaskStatusManager manager = ToolsLocator.getTaskStatusManager();
116 211
        this.taskStatus = manager.createDefaultSimpleTaskStatus(readerFactory.getName());
117 212

  
118
        counterNewsOIDs = 0;
119

  
120 213
        File file = getParameters().getFile();
121 214
        resource = this.createResource(
122 215
                FileResource.NAME,
......
124 217
        );
125 218

  
126 219
        resource.addConsumer(this);
220
        if( storeServices != null ) {
221
            initializeFeatureTypes();
222
        }
223
    }
224

  
225
    private void setStoreServices(FeatureStoreProviderServices storeServices) throws InitializeException {
226
        this.store = storeServices;
127 227
        initializeFeatureTypes();
128 228
    }
129

  
229
    
130 230
    @Override
131 231
    public SimpleSequentialReaderStoreParameters getParameters() {
132 232
        return (SimpleSequentialReaderStoreParameters) super.getParameters();
......
163 263
        }
164 264
        this.data = new ArrayList<>();
165 265
        resource.setData(new HashMap());
166
        counterNewsOIDs = 0;
167 266
        try {
168 267
            loadFeatures();
169 268
        } catch (RuntimeException e) {
......
227 326

  
228 327
    @Override
229 328
    public Object createNewOID() {
230
        return counterNewsOIDs++;
329
        throw new UnsupportedOperationException();
231 330
    }
232 331

  
233 332
    protected void initializeFeatureTypes() throws InitializeException {
......
327 426
        return resource;
328 427
    }
329 428

  
429
    private String[] split(String value, String separators) {
430
        int firstSeparatorPosition = 1000000;
431
        Character sep = null;
432
        for (char ch : separators.toCharArray()) {
433
            int pos = value.indexOf(ch);
434
            if( pos>0 && pos<firstSeparatorPosition ) {
435
                sep = ch;
436
                firstSeparatorPosition = pos;
437
            }
438
        }
439
        if( sep == null ) {
440
            return new String[] { value };
441
        }
442
        return value.split("["+sep+"]");
443
    }
444
    
330 445
    private class FieldTypeParser {
331 446

  
332 447
        public String name = null;
333 448
        public int type = DataTypes.STRING;
334 449
        public int size = 0;
335
        public boolean allowNulls = true;
336 450
        public int geomType = Geometry.TYPES.GEOMETRY;
337 451
        public int geomSubtype = Geometry.SUBTYPES.GEOM2D;
452
        public Map<String,String> tags = new HashMap<>();
453
        public Map<String,String> assignments = new HashMap<>();
338 454

  
339 455
        private String typename = "string";
340 456

  
......
349 465
        @SuppressWarnings("UseSpecificCatch")
350 466
        public boolean parse(String value) {
351 467
            String[] args;
352
            if ( value.contains(":") ) {
353
                args = value.split(":");
354
            } else if ( value.contains("/") ) {
355
                args = value.split("/");
356
            } else if ( value.contains("__") ) {
468
            if ( value.contains("__") ) {
357 469
                args = value.split("__");
358 470
            } else {
359
                this.name = value;
360
                return true;
361
            }
362
            
471
                args = split(value, ":/#@!;-");
472
                if( args.length == 1 ) {
473
                    this.name = value;
474
                    return true;
475
                }
476
            }                        
363 477
            int n = 0;
364 478
            this.name = args[n++];
365 479
            if( n >= args.length ) {
......
406 520
                            LOGGER.warn("Ignore incorrect field size for field " + value + " in '"+getProviderName()+"' file '" + getFullFileName() + "'.", ex);
407 521
                        }
408 522
                        break;
409
                    case "geomtype":
410
                        this.geomType = GeometryUtils.getGeometryType(args[n++]);
411
                        break;
412
                    case "geomsubtype":
413
                        this.geomSubtype = GeometryUtils.getGeometrySubtype(args[n++]);
414
                        break;
415
                    case "allownulls":
416
                        this.allowNulls = true;
417
                        break;
418
                    case "notallownulls":
419
                        this.allowNulls = false;
420
                        break;
523
                    case "tag": {
524
                            String x = args[n++];
525
                            int pos = x.indexOf("=");
526
                            if( pos < 0 ) {
527
                                this.tags.put(x, null);
528
                            } else {
529
                                this.tags.put( 
530
                                        StringUtils.substring(x, 0, pos),
531
                                        StringUtils.substring(x, pos+1)
532
                                );
533
                            }
534
                            break;
535
                        }
536
                    case "set": {
537
                            String x = args[n++];
538
                            int pos = x.indexOf("=");
539
                            if( pos < 0 ) {
540
                                this.assignments.put(x, null);
541
                            } else {
542
                                this.assignments.put( 
543
                                        StringUtils.substring(x, 0, pos),
544
                                        StringUtils.substring(x, pos+1)
545
                                );
546
                            }
547
                            break;
548
                        }
421 549
                    default:
422 550
                        LOGGER.warn("Illegal argumente '"+option+"' in '"+getProviderName()+"' file '" + getFullFileName() + "'.");
423 551
                }
......
473 601
        //
474 602
        // Una vez ya sabemos los tipos de datos rellenamos el feature-type
475 603
        //
604
        DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager();
476 605
        for (FieldTypeParser fieldType : fieldTypes) {
477 606
            EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type);
478 607
            fad.setSize(fieldType.size);
479
            fad.setAllowNull(fieldType.allowNulls);
480 608
            if (fieldType.type == DataTypes.GEOMETRY ) {
481 609
                fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype);
482 610
                if( fType.getDefaultGeometryAttributeName() == null ) {
483 611
                    fType.setDefaultGeometryAttributeName(fieldType.name);
484 612
                }
485 613
            }
614
            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {
615
                try {
616
                    switch(entry.getKey().toLowerCase()) {
617
                        case "hidden":
618
                            fad.setHidden((boolean) dataTypesManager.coerce(DataTypes.BOOLEAN, entry.getValue()));
619
                            break;
620
                        case "allownull":
621
                            fad.setAllowNull((boolean) dataTypesManager.coerce(DataTypes.BOOLEAN, entry.getValue()));
622
                            break;
623
                        case "isautomatic":
624
                        case "automatic":
625
                            fad.setIsAutomatic((boolean) dataTypesManager.coerce(DataTypes.BOOLEAN, entry.getValue()));
626
                            break;
627
                        case "time":
628
                        case "istime":
629
                            fad.setIsTime((boolean) dataTypesManager.coerce(DataTypes.BOOLEAN, entry.getValue()));
630
                            break;
631
                        case "profile":
632
                            fad.setDataProfileName((String) dataTypesManager.coerce(DataTypes.STRING, entry.getValue()));
633
                            break;
634
                        case "group":
635
                            fad.setGroup((String) dataTypesManager.coerce(DataTypes.STRING, entry.getValue()));
636
                            break;
637
                        case "description":
638
                            fad.setDescription((String) dataTypesManager.coerce(DataTypes.STRING, entry.getValue()));
639
                            break;
640
                        case "label":
641
                            fad.setLabel((String) dataTypesManager.coerce(DataTypes.STRING, entry.getValue()));
642
                            break;
643
                        case "expression":
644
                            // Los campos calculados los procesamos en una segunda
645
                            // pasada, cuando ya estan definidos el resto de los campos
646
                            // ya que pueden requerir campos que aun no se han definido.
647
                            break;
648
                        case "size":
649
                            fad.setSize((int) dataTypesManager.coerce(DataTypes.INT, entry.getValue()));
650
                            break;
651
                        case "precision":
652
                            fad.setPrecision((int) dataTypesManager.coerce(DataTypes.INT, entry.getValue()));
653
                            break;
654
                        case "order":
655
                            fad.setOrder((int) dataTypesManager.coerce(DataTypes.INT, entry.getValue()));
656
                            break;
657
                        case "interval":
658
                            fad.setInterval((Interval) dataTypesManager.coerce(DataTypes.INTERVAL, entry.getValue()));
659
                            break;
660
                        case "geomtype":
661
                        case "geometrytype":
662
                            fad.setGeometryType(entry.getValue());
663
                            break;
664
                        case "srs":
665
                            fad.setSRS(entry.getValue());
666
                            break;
667
                    }
668
                } catch (Exception ex) {
669
                    LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex);
670
                }
671
            }
672
            Tags tags = fad.getTags();
673
            for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) {
674
                tags.set(entry.getKey(), entry.getValue());
675
            }
486 676
        }
677
        // Processamos ahora los campos calculados
678
        for (FieldTypeParser fieldType : fieldTypes) {
679
            EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name);
680
            for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) {
681
                try {
682
                    switch(entry.getKey().toLowerCase()) {
683
                        case "expression":
684
                            fad.setFeatureAttributeEmulator((String) dataTypesManager.coerce(DataTypes.STRING, entry.getValue()));
685
                            break;
686
                    }
687
                } catch (Exception ex) {
688
                    LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+getFullFileName()+"'.", ex);
689
                }
690
            }
691
        }
692

  
487 693
        String[] pointDimensionNames = this.getParameters().getPointDimensionNames();
488 694
        if ( pointDimensionNames != null ) {
489 695
            PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames);
......
605 811

  
606 812
    }
607 813

  
608

  
609 814
    private void loadFeatures() throws IOException, DataException,
610 815
            CoercionException, CloneNotSupportedException {
611
        SimpleSequentialReader reader = null;
612 816

  
817
        ReaderData theReaderData = this.readerData;
818
        if( theReaderData==null ) {
819
            theReaderData = new ReaderData();
820
            SimpleSequentialReader reader = this.readerFactory.createReader(this.getParameters());
821
            try {
822
                loadFeatures(reader, theReaderData);
823

  
824
                for(SimpleSequentialReader childReader : reader.getChildren() ) {
825
                    ReaderData childData = new ReaderData();
826
                    loadFeatures(childReader, childData);
827
                    this.childrenData.add(childData);
828
                }
829
                this.readerData = theReaderData;
830
            } finally {
831
                reader.close();
832
            }
833
        }
834
        this.name = theReaderData.getName();
835
        FeatureStoreProviderServices store = this.getStoreServices();
836
        store.setFeatureTypes(theReaderData.getFeatureTypes(), theReaderData.getDefaultFeatureType());
837
        this.need_calculate_envelope = theReaderData.getNeedCalculateEnvelope();
838
        this.data = theReaderData.getFeatures();
839

  
840
    }
841
    
842
    private void loadFeatures(SimpleSequentialReader reader, ReaderData readerData) throws IOException, DataException,
843
            CoercionException, CloneNotSupportedException {
844

  
613 845
        try {
614 846
            List<String> headers;
615
            FeatureStoreProviderServices store = this.getStoreServices();
616 847

  
848
            readerData.setName(reader.getName());
849
            
617 850
            boolean ignore_errors = getParameters().getIgnoreErrors();
618 851

  
619
            reader = this.readerFactory.createReader(this.getParameters());
620
            this.name = reader.getName();
621
            
622 852
            headers = getParameters().getFieldNames();
623 853
            if ( headers == null ) {
624 854
                headers = reader.getFieldNames();
......
626 856

  
627 857
            // Initialize the feature types
628 858
            EditableFeatureType edftype = this.getFeatureType(headers, automaticDetectionOfTypes(reader));
859

  
860
            edftype.setLabel(reader.getLabel());
861
            edftype.setDescription(reader.getDescription());
862
            Map<String, String> tagsReader = reader.getTags();
863
            if( tagsReader!=null ) {
864
                Tags tagsType = edftype.getTags();
865
                for (Map.Entry<String, String> tag : tagsReader.entrySet()) {
866
                    tagsType.set(tag.getKey(), tag.getValue());
867
                }
868
            }
869
            
629 870
            FeatureType ftype = edftype.getNotEditableCopy();
630 871
            List<FeatureType> ftypes = new ArrayList<>();
631 872
            ftypes.add(ftype);
632
            store.setFeatureTypes(ftypes, ftype);
873
            readerData.setFeatureTypes(ftypes, ftype);
633 874

  
634 875
            Coercion coercion[] = new Coercion[ftype.size()];
635 876
            int sizes[] = new int[ftype.size()];
......
645 886
                    }
646 887
                }
647 888
            }
889
           
648 890
            if ( ftype.getDefaultGeometryAttributeName() != null ) {
649
                this.need_calculate_envelope = true;
891
                readerData.setNeedCalculateEnvelope(true);
650 892
            }
651 893

  
652 894
            Locale locale = getParameters().getLocale();
......
663 905
                taskStatus.setCurValue(++count);
664 906
                FeatureProvider feature = this.createFeatureProvider(ftype);
665 907
                for ( int i = 0; i < row.size(); i++ ) {
666
                    rawvalue = row.get(i);
667
                    if( rawvalue instanceof String && StringUtils.isBlank((String)rawvalue) ) {
668
                        rawvalue = null;
669
                    }
670 908
                    try {
909
                        if( ftype.get(i).isComputed() ) {
910
                            continue;
911
                        }
912
                        rawvalue = row.get(i);
913
                        if( rawvalue instanceof String && StringUtils.isBlank((String)rawvalue) ) {
914
                            rawvalue = null;
915
                        }
671 916
                        Object value;
672 917
                        if ( locale != null && coercion[i] instanceof CoercionWithLocale ) {
673 918
                            value = ((CoercionWithLocale) (coercion[i])).coerce(rawvalue, locale);
......
681 926
                                sizes[i] = x;
682 927
                            }
683 928
                        }
684
                    } catch (RuntimeException ex) {
929
                    } catch (Exception ex) {
685 930
                        if ( !ignore_errors ) {
686 931
                            throw ex;
687 932
                        }
688 933
                        if ( count_errors++ < 10 ) {
689
                            LOGGER.warn("Can't load value of attribute " + i + " in row " + count + ".", ex);
934
                            LOGGER.warn("Can't load value of attribute " + i+"/"+row.size()+"/"+ftype.size()+ " in row " + count + " in " +readerData.getName()+ ".", ex);
690 935
                        }
691 936
                        if ( count_errors == 10 ) {
692 937
                            LOGGER.info("Too many errors, suppress messages.");
693 938
                        }
694 939
                    }
695 940
                }
696
                this.addFeatureProvider(feature);
941
                readerData.addFeatureProvider(feature);
697 942
                row = reader.read();
698 943
            }
699 944
            for ( int i = 0; i < ftype.size(); i++ ) {
700 945
                if ( sizes[i] > 0 ) {
701 946
                    EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i));
702
                    efad.setSize(sizes[i]);
947
                    if( efad.getSize()<sizes[i] ) {
948
                        efad.setSize(sizes[i]);
949
                    }
703 950
                }
704 951
            }
705 952
            // Volvemos a asignar al store el featuretype, ya que puede
......
707 954
            ftype = edftype.getNotEditableCopy();
708 955
            ftypes = new ArrayList<>();
709 956
            ftypes.add(ftype);
710
            store.setFeatureTypes(ftypes, ftype);
957
            readerData.setFeatureTypes(ftypes, ftype);
711 958

  
712 959
            taskStatus.terminate();
713 960
        } catch (Exception ex) {
714 961
            LOGGER.warn("Can't load features from '"+getProviderName()+"' file '" + getFullFileName() + "'.", ex);
715
        } finally {
716
            if ( reader != null ) {
717
                try {
718
                    reader.close();
719
                } catch (Exception ex) {
720
                    // Do nothing
721
                }
722
            }
723

  
724 962
        }
725 963
    }
726 964
    
......
762 1000
        return super.hasDynValue(name);
763 1001
    }
764 1002

  
1003
    @Override
1004
    public DataStore getChild(String name) {
1005
        for (ReaderData childData : this.childrenData) {
1006
            if( StringUtils.equalsIgnoreCase(name, childData.getName()) ) {
1007
                try {
1008
                    return childData.getStore();
1009
                } catch (InitializeException ex) {
1010
                    throw new RuntimeException("Can't acces to child '"+name+"'", ex);
1011
                }
1012
            }
1013
        }        
1014
        return null;
1015
    }
765 1016
    
1017
    @Override
1018
    public Iterator<DataStore> getChildren() {
1019
        if( this.childrenData==null || this.childrenData.isEmpty() ) {
1020
            return Collections.EMPTY_LIST.iterator();
1021
        }
1022
        try {
1023
            List<DataStore> children = new ArrayList<>();
1024
            for (ReaderData childData : this.childrenData) {
1025
                    children.add(childData.getStore());
1026
            }
1027
            return children.iterator();
1028
        } catch (InitializeException ex) {
1029
            throw new RuntimeException("Can't access to children", ex);
1030
        }
1031
    }
1032

  
1033
    @Override
1034
    public boolean hasChildren() {
1035
        return this.childrenData!=null && !this.childrenData.isEmpty();
1036
    }
1037
    
766 1038
}

Also available in: Unified diff