Statistics
| Revision:

svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.plugin / org.gvsig.xml2db / org.gvsig.xml2db.lib / org.gvsig.xml2db.lib.impl / src / main / java / org / gvsig / xml2db / lib / impl / StructureExtractorImpl.java @ 47360

History | View | Annotate | Download (22.4 KB)

1
/*
2
 * To change this license header, choose License Headers in Project Properties.
3
 * To change this template file, choose Tools | Templates
4
 * and open the template in the editor.
5
 */
6
package org.gvsig.xml2db.lib.impl;
7

    
8
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLAttributeInfoImpl;
9
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLTableInfoImpl;
10
import java.io.File;
11
import java.io.FileNotFoundException;
12
import java.io.IOException;
13
import java.io.InputStream;
14
import java.io.Reader;
15
import java.nio.charset.Charset;
16
import java.util.ArrayList;
17
import java.util.List;
18
import java.util.Locale;
19
import javax.xml.parsers.SAXParser;
20
import javax.xml.parsers.SAXParserFactory;
21
import org.apache.commons.lang3.ArrayUtils;
22
import org.apache.commons.lang3.StringUtils;
23
import org.apache.tika.utils.CharsetUtils;
24
import org.cresques.cts.IProjection;
25
import org.gvsig.fmap.crs.CRSFactory;
26
import org.gvsig.fmap.dal.DALLocator;
27
import org.gvsig.fmap.dal.DataManager;
28
import org.gvsig.fmap.dal.DataTypes;
29
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;
30
import org.gvsig.fmap.dal.feature.EditableFeatureType;
31
import org.gvsig.fmap.dal.feature.EditableForeingKey;
32
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;
33
import org.gvsig.tools.ToolsLocator;
34
import org.gvsig.tools.dynobject.DynField_v2;
35
import org.gvsig.tools.i18n.I18nManager;
36
import org.gvsig.tools.task.SimpleTaskStatus;
37
import org.gvsig.xml2db.lib.api.xmlinfo.XMLAttributeInfo;
38
import org.gvsig.xml2db.lib.api.xmlinfo.XMLInfo;
39
import org.gvsig.xml2db.lib.api.xmlinfo.XMLTableInfo;
40
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLInfoImpl;
41
import org.slf4j.Logger;
42
import org.slf4j.LoggerFactory;
43
import org.xml.sax.Attributes;
44
import org.xml.sax.InputSource;
45
import org.xml.sax.Locator;
46
import org.xml.sax.SAXException;
47
import org.xml.sax.helpers.DefaultHandler;
48

    
49
/**
50
 *
51
 * @author jjdelcerro
52
 */
53
public class StructureExtractorImpl {
54

    
55
        private static final Logger LOGGER = LoggerFactory.getLogger(StructureExtractorImpl.class);
56

    
57
    @SuppressWarnings("UseSpecificCatch")
58
    private void extractTags(XMLInfoImpl xmlinfo, Reader reader, SimpleTaskStatus status) {
59
        if( reader == null ) {
60
            throw new IllegalArgumentException("reader is null");
61
        }
62
        try {
63
            final DataManager dataManager = DALLocator.getDataManager();
64
            I18nManager i18n = ToolsLocator.getI18nManager();
65
            status.message(i18n.getTranslation("_Reading_xml")+" 1/5");
66
            status.setRangeOfValues(0, xmlinfo.getCountLines());
67
            
68
            SAXParserFactory spf = SAXParserFactory.newInstance();
69
            spf.setNamespaceAware(true);
70
            SAXParser saxParser = spf.newSAXParser();
71
            InputSource is = new InputSource(reader);
72
            
73
            List<String> path = new ArrayList<>();
74
            
75
            saxParser.parse(is, new DefaultHandler() {
76
                private Locator locator;
77
                int size;
78
                int refreshInterval = 1;
79
                StringBuilder chars = new StringBuilder();
80
                
81
                @Override
82
                public void setDocumentLocator(Locator locator) {
83
                    this.locator = locator;
84
                }
85
                
86
                @Override
87
                public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
88
                    int line = this.locator.getLineNumber();
89
                    int column = this.locator.getColumnNumber()-2-localName.length();
90

    
91
                    if(line % refreshInterval == 0) {
92
                        status.setCurValue(line);
93
                    }
94
                    
95
                    if(line > 100000){
96
                        refreshInterval = 10000;
97
                    } else if(line > 10000){
98
                        refreshInterval = 1000;
99
                    } else if(line > 1000){
100
                        refreshInterval = 100;
101
                    } else if(line > 100){
102
                        refreshInterval = 10;
103
                    }
104

    
105

    
106
                    String idvalue = dataManager.createUniqueID();
107
                    
108
                    path.add(localName);
109
                    String path_s = StringUtils.join(path, "/");
110
                    XMLAttributeInfoImpl info = xmlinfo.getTag(path_s);
111
                    if( info == null ) {
112
                        info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s);
113
                        xmlinfo.addTag(info);
114
                        
115
                    }
116
                    if( path.size()>1 ) {
117
                        List<String> parentpath = path.subList(0, path.size()-1);
118
                        String parentpath_s = StringUtils.join(parentpath, "/");
119
                        XMLAttributeInfoImpl parentinfo = xmlinfo.getTag(parentpath_s);
120
                        parentinfo.incrChildCount(localName);
121
                        parentinfo.setLastChildID(localName, idvalue);
122
                    }
123
                    
124
                    for (int i = 0; i < attributes.getLength(); i++) {
125
                        String name = attributes.getLocalName(i);                        
126
                        String value = attributes.getValue(i);
127
                        String idvalueChild = dataManager.createUniqueID();
128
                        XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name);
129
                        if( infoChild == null ) {
130
                            infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name);
131
                            xmlinfo.addTag(infoChild);
132
                        }
133
                        info.incrChildCount(infoChild.getName());
134
                        info.setLastChildID(infoChild.getName(), idvalueChild);
135
                        infoChild.addValue(value);
136
                        if( xmlinfo.getSrid() ==  null && StringUtils.containsIgnoreCase(infoChild.getName(), "srid") ) {
137
                            IProjection proj = getProjection(value);
138
                            if( proj!=null ) {
139
                                xmlinfo.setSrid(proj);
140
                            }
141
                        }
142
                    }
143
                    chars.setLength(0);
144
                }
145
                
146
                @Override
147
                public void endElement(String uri, String localName, String qName) throws SAXException {
148
                    int line = this.locator.getLineNumber();
149

    
150
//                    status.setCurValue(line);
151

    
152
                    String path_s = StringUtils.join(path, "/");
153
                    XMLAttributeInfoImpl info = xmlinfo.getTag(path_s);
154
                    if( info == null ) {
155
                        info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s);
156
                        xmlinfo.addTag(info);
157
                    }
158

    
159
                    XMLAttributeInfoImpl parentinfo = null;
160
                    if( path.size()>1 ) {
161
                        List<String> parentpath = path.subList(0, path.size()-1);
162
                        String parentpath_s = StringUtils.join(parentpath, "/");
163
                        parentinfo = xmlinfo.getTag(parentpath_s);
164
                    }
165
                    
166
                    if( info.hasChilds() || (parentinfo != null && (parentinfo.getChildsCount(localName)>1))) {
167
                        String value = this.chars.toString();
168
                        if( StringUtils.isNotBlank(value) ) {
169
                            String name = info.getName()+"$v";                        
170
                            String idvalueChild = dataManager.createUniqueID();
171
                            XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name);
172
                            if( infoChild == null ) {
173
                                infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name);
174
                                xmlinfo.addTag(infoChild);
175
                            }
176
                            info.incrChildCount(infoChild.getName());
177
                            info.setLastChildID(infoChild.getName(), idvalueChild);
178
                            infoChild.addValue(value);
179
                        }
180
                    } else {
181
                        String value = this.chars.toString();
182
                        info.addValue(value);
183
                        if( StringUtils.containsIgnoreCase(info.getName(), "srid") ) {
184
                            IProjection proj = getProjection(value);
185
                            if( proj!=null ) {
186
                                xmlinfo.setSrid(proj);
187
                            }
188
                        }
189
                    }
190
                    info.consolidateChildCounters();
191
//                    if( StringUtils.equalsIgnoreCase("LUGAR_CIRCULABA", info.getName()) ) {
192
//                        System.out.println("Oh!");
193
//                    }
194
                    
195
                    path.remove(path.size()-1);
196
                    chars.setLength(0);
197
                }
198
                
199
                @Override
200
                public void characters(char[] ch, int start, int length) throws SAXException {
201
                    int line = this.locator.getLineNumber();
202

    
203
//                    status.setCurValue(line);
204
                    this.chars.append(ch, start, length);
205
                }
206
                
207
                
208
            });
209
        } catch (Exception ex) {
210
            throw new RuntimeException("Can't extract tags.", ex);
211
        }
212
    }
213
    
214
    private IProjection getProjection(String value) {
215
        if( StringUtils.isBlank(value) ) {
216
            return null;
217
        }
218
        IProjection proj = null;
219
        try {
220
            proj = CRSFactory.getCRS(value);
221
        } catch(Throwable t) {
222
            
223
        }
224
        if( proj != null ) {
225
            return proj;
226
        }
227
        try {
228
            proj = CRSFactory.getCRS("EPSG:"+value);
229
        } catch(Throwable t) {
230
            
231
        }
232
        return proj;
233
    }
234
    
235
    public XMLInfo extractStructure(File xml, Charset charset, IProjection projection, Locale locale, SimpleTaskStatus status) throws FileNotFoundException, IOException {
236
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
237
        xmlinfo.setLocale(locale);
238
        xmlinfo.setSrid(projection);
239
        long count = Xml2dbCommons.countLines(xml, charset, status);
240
        xmlinfo.setCountLines(count);
241
        InputSource is = Xml2dbCommons.openReader(xml,charset);
242
        return extractStructure(is, xmlinfo, status);
243
    }
244
    
245
    public XMLInfo extractStructure(InputStream xml, Charset charset, IProjection projection, Locale locale, SimpleTaskStatus status) throws IOException  {
246
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
247
        xmlinfo.setLocale(locale);
248
        xmlinfo.setSrid(projection);
249
//        long count = Xml2dbCommons.countLines(xml, charset, status);
250
        xmlinfo.setCountLines(-1);
251
        InputSource is = Xml2dbCommons.openReader(xml, charset);
252
        return extractStructure(is, xmlinfo, status);
253
    }
254
    
255
    public XMLInfo extractStructure(Reader reader, IProjection projection, Locale locale, SimpleTaskStatus status) {
256
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
257
        xmlinfo.setLocale(locale);
258
        xmlinfo.setSrid(projection);
259
        InputSource is = new InputSource(reader);
260
        return extractStructure(is, xmlinfo, status);
261
    }
262
    
263
    public XMLInfo extractStructure(InputSource is, XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
264
        
265
        if( xmlinfo.getCharset()==null ) {
266
            xmlinfo.setCharset(CharsetUtils.forName(is.getEncoding()));
267
        }
268
        
269
        extractTags(xmlinfo, is.getCharacterStream(), status);
270
        
271
        buildTablesFromTags(xmlinfo, status);
272

    
273
        removeDuplicateTableNames(xmlinfo, status);
274
        
275
        createPrimaryKeyAndForeignKeys(xmlinfo, status);
276
        
277
        createFeatureTypes(xmlinfo, status);
278

    
279
        return xmlinfo;
280
    }
281
    
282
    private String plural(String s) {
283
        int l = s.length();
284
        char ch = s.substring(l-1,l).toLowerCase().charAt(0);
285
        if( ch=='a' || ch=='e' || ch=='i' || ch=='o' || ch=='u' ) {
286
            return s+"s";
287
        }
288
        return s+"es";
289
    }
290

    
291
    private void buildTablesFromTags(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
292
        I18nManager i18n = ToolsLocator.getI18nManager();
293
        status.message(i18n.getTranslation("_Identifying_tables")+" 2/5");
294
        status.setRangeOfValues(0, xmlinfo.getTagsPaths().size());
295
        
296
        for (String tagPath : xmlinfo.getTagsPaths()) {
297
            XMLAttributeInfoImpl tag1Info = xmlinfo.getTagInfo(tagPath);
298
//            System.out.println(tag1Info.getPath());
299
            XMLTableInfoImpl tableInfo = new XMLTableInfoImpl(tagPath, tag1Info);
300
            for (String tagPath2 : xmlinfo.getTagsPaths()) {
301
                if( tagPath.equals(tagPath2) ) {
302
                    continue;
303
                }
304
                if( tagPath2.startsWith(tagPath+"/") ) {
305
                    String fieldName = tagPath2.substring(tagPath.length()+1);
306
                    if( !fieldName.contains("/") ) {
307
                        XMLAttributeInfoImpl info = xmlinfo.getTag(tagPath2);
308
                        tableInfo.add(info);
309
                    }
310
                }
311
            }
312
            if( !tableInfo.isEmpty() ) {
313
                xmlinfo.addTable(tableInfo);
314
            }
315
            status.incrementCurrentValue();
316
        }
317
    }
318
    
319
    private void removeDuplicateTableNames(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
320
        I18nManager i18n = ToolsLocator.getI18nManager();
321
        status.message(i18n.getTranslation("_Fixing_table_names")+" 3/5");
322
        status.setRangeOfValues(0, xmlinfo.size());
323

    
324
        // Tratamos de corregir nombres duplicados en las tablas.
325
        for (XMLTableInfo tableInfo01 : xmlinfo) {
326
            XMLTableInfoImpl tableInfo1 = (XMLTableInfoImpl) tableInfo01;
327
            
328
            status.message(i18n.getTranslation("_Fixing_table_names")+" 3/5 ("+tableInfo1.getName()+")");
329

    
330
            boolean renombrar = false;
331
            for (XMLTableInfo tableInfo02 : xmlinfo) {
332
                XMLTableInfoImpl tableInfo2 = (XMLTableInfoImpl) tableInfo02;
333
                if( tableInfo1 == tableInfo2 ) {
334
                    continue;
335
                }
336
                if( StringUtils.equalsIgnoreCase(tableInfo1.getName(), tableInfo2.getName()) ) {
337
                    // FIXME: solo usa los dos ultimos nombres, podrian coincidir igualmente!!
338
                    String[] path_ss = tableInfo2.getPath().split("/");
339
                    int l = path_ss.length;
340
                    String name = path_ss[l-1]+"_"+path_ss[l-2];
341
                    tableInfo2.rename(name);
342
                    renombrar = true;
343
//                    System.out.println("###: Tablas con el mismo nombre, renombrado a "+name);
344
//                    System.out.println("###: "+tableInfo1.getPath());
345
//                    System.out.println("###: "+tableInfo2.getPath());
346
                }
347
            }
348
            if( renombrar ) {
349
                // FIXME: solo usa los dos ultimos nombres, podrian coincidir igualmente!!
350
                String[] path_ss = tableInfo1.getPath().split("/");
351
                int l = path_ss.length;
352
                String name = path_ss[l-1]+"_"+path_ss[l-2];
353
                tableInfo1.rename(name);
354
            }
355
            status.incrementCurrentValue();
356
        }
357
    }
358
    
359
    private void createPrimaryKeyAndForeignKeys(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
360
        I18nManager i18n = ToolsLocator.getI18nManager();
361
        status.message(i18n.getTranslation("_Identifying_foreign_keys")+" 4/5");
362
        status.setRangeOfValues(0, xmlinfo.size());
363

    
364
        for (XMLTableInfo tableInfo0 : xmlinfo) {
365
            XMLTableInfoImpl tableInfo = (XMLTableInfoImpl) tableInfo0;
366
            
367
            status.message(i18n.getTranslation("_Identifying_foreign_keys")+" 4/5 ("+tableInfo.getName()+")");
368

    
369
            String[] fieldkeys = tableInfo.getPath().split("/");
370
            for (int i = 0; i < fieldkeys.length; i++) {
371
                String fieldkey = fieldkeys[i];
372
                if( i==fieldkeys.length-1 ) {
373
                    tableInfo.add(new XMLAttributeInfoImpl(xmlinfo.getLocale(), "$ID_"+tableInfo.getName(), DataTypes.INT).setPk(true));
374
                } else { 
375
                    // TODO es una relacion 1:1
376
                    XMLTableInfo tableInfo2 = xmlinfo.getTableByPath(StringUtils.join(ArrayUtils.subarray(fieldkeys, 0, i+1),"/"));
377
                    tableInfo.add(new XMLAttributeInfoImpl(xmlinfo.getLocale(), "$ID_"+tableInfo2.getName(), DataTypes.INT)
378
                            .setFk(true)
379
                            .setFkCodeName("$ID_"+fieldkey)
380
                            .setFkTableName(fieldkey)
381
                    );
382
                }
383
            }
384

    
385
            for (XMLAttributeInfo field0 : tableInfo) {
386
                XMLAttributeInfoImpl field = (XMLAttributeInfoImpl) field0;
387
                if( xmlinfo.existsTableByPath(tableInfo.getPath()+"/"+field.getName()) ) {
388
                    XMLTableInfoImpl fktableInfo = xmlinfo.getTableByPath(tableInfo.getPath()+"/"+field.getName());
389
                    int count = tableInfo.getMaxCountChild(field.getName());
390
                    if( count<1 ) {
391
                        // Do nothing
392
                    } else if( count>1 ) {
393
                        // TODO es una relacion 1:n
394
                        field.setAggregate(true);
395
                        field.setFkCodeName("$ID_"+tableInfo.getName());
396
                        field.setFkTableName(fktableInfo.getName());
397
                        field.setSize(45);
398
                        field.setType(DataTypes.LIST);
399
                    } else {
400
                        // TODO es una relacion 1:1
401
                        field.setFk(true);
402
                        field.setFkCodeName("$ID_"+fktableInfo.getName());
403
                        field.setFkTableName(fktableInfo.getName());
404
                        field.setSize(0);
405
                        field.setType(DataTypes.INT);
406
                    }
407
                }
408
            }            
409
            tableInfo.sort();
410
//            System.out.println(tableInfo);
411
            status.incrementCurrentValue();
412
        }
413
    }
414
    
415
    private void createFeatureTypes(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
416
        I18nManager i18n = ToolsLocator.getI18nManager();
417
        status.message(i18n.getTranslation("_Creating_table_definitions")+" 5/5");
418
        status.setRangeOfValues(0, xmlinfo.size());
419

    
420
        DataManager dataManager = DALLocator.getDataManager();
421

    
422
        for (XMLTableInfo tableInfo0 : xmlinfo) {
423
            XMLTableInfoImpl tableInfo = (XMLTableInfoImpl) tableInfo0;
424

    
425
            status.message(i18n.getTranslation("_Creating_table_definitions")+" 5/5 ("+tableInfo.getName()+")");
426
            
427
            EditableFeatureType ft = dataManager.createFeatureType();
428
            ft.setLabel(tableInfo.getName());
429
            ft.getTags().set("xml2db.tablename", tableInfo.getName());
430
            ft.getTags().set("vcsgis.storename", tableInfo.getName());
431
            ft.getTags().set("xml2db.path", tableInfo.getPath());
432
            for (XMLAttributeInfo attrinfo0 : tableInfo) {
433
                XMLAttributeInfoImpl attrinfo = (XMLAttributeInfoImpl) attrinfo0;
434
                if( ft.get(attrinfo.getName())!=null ) {
435
                    LOGGER.warn("Duplicated attribute '"+attrinfo.getName()+"' in '"+tableInfo.getName()+"'.");
436
                }
437
                if(  attrinfo.isAggregate() ) {
438
                    continue;
439
                }
440
                EditableFeatureAttributeDescriptor attrdesc;
441
                if( attrinfo.getName().startsWith("$ID") ) {
442
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
443
                    attrdesc.setLabel("Id. "+attrinfo.getName().substring(4));
444
                    attrdesc.setSize(attrinfo.getSize());
445
                } else if( attrinfo.getName().endsWith("$v") ) {
446
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
447
                    attrdesc.setLabel(StringUtils.left(attrinfo.getName(),attrinfo.getName().length()-2));
448
                    attrdesc.setSize(attrinfo.getSize());
449
                } else {
450
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
451
                    attrdesc.setSize(attrinfo.getSize());
452
                    attrdesc.setPrecision(attrinfo.getPrecision());
453
                    attrdesc.setScale(attrinfo.getScale());
454
                    if( attrdesc.getType()==DataTypes.GEOMETRY ) {
455
                        attrdesc.setGeometryType(attrinfo.getGeometryType());
456
                        attrdesc.setSRS(xmlinfo.getSrid());
457
                    }
458
                }
459
                attrdesc.setLocale(xmlinfo.getLocale());
460
                attrdesc.setIsPrimaryKey(attrinfo.isPk());
461
                if( attrinfo.isFk() ) {
462
                    EditableForeingKey fk = attrdesc.getForeingKey();
463
                    fk.setForeingKey(true);
464
                    fk.setTableName(attrinfo.getFkTableName());
465
                    fk.setCodeName(attrinfo.getFkCodeName());
466
                    fk.setClosedList(false);
467
                    fk.setLabelFormula("\""+attrinfo.getFkCodeName()+"\"");
468
                    attrdesc.setRelationType(DynField_v2.RELATION_TYPE_COLLABORATION);
469
                    attrdesc.setIsIndexed(true);
470
                }
471
            }
472
            FeatureAttributeDescriptor pk = ft.getPrimaryKey()[0];
473
            for (XMLAttributeInfo attrinfo0 : tableInfo) {
474
                XMLAttributeInfoImpl attrinfo = (XMLAttributeInfoImpl) attrinfo0;
475
                if(  attrinfo.isAggregate() ) {
476
                    EditableFeatureAttributeDescriptor attrdesc = ft.add("$List_"+attrinfo.getName(), attrinfo.getType());
477
                    attrdesc.setLabel(plural(attrinfo.getName()));
478
                    attrdesc.getTags().set("dal.relatedfeatures.table", attrinfo.getFkTableName());
479
                    attrdesc.getTags().set("dal.relatedfeatures.unique.field.name", attrinfo.getFkCodeName());
480
                    attrdesc.getTags().set("dynform.label.empty", true);
481
                    attrdesc.getTags().set("dynform.resizeWeight", 100);
482
                    attrdesc.setFeatureAttributeEmulator(
483
                        "SELECT * FROM \""+attrinfo.getFkTableName() + "\" WHERE ( (\""+pk.getName()+"\") = (\""+attrinfo.getFkTableName() + "\".\""+pk.getName()+"\") )"                    
484
                    );
485
                    attrdesc.setRelationType(DynField_v2.RELATION_TYPE_AGGREGATE);
486
                }
487
            }
488
            tableInfo.setFeatureType(ft);
489
            status.incrementCurrentValue();
490
        }
491
    }
492
}