Statistics
| Revision:

svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.plugin / org.gvsig.xml2db / org.gvsig.xml2db.lib / org.gvsig.xml2db.lib.impl / src / main / java / org / gvsig / xml2db / lib / impl / StructureExtractorImpl.java @ 47352

History | View | Annotate | Download (22.4 KB)

1
/*
2
 * To change this license header, choose License Headers in Project Properties.
3
 * To change this template file, choose Tools | Templates
4
 * and open the template in the editor.
5
 */
6
package org.gvsig.xml2db.lib.impl;
7

    
8
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLAttributeInfoImpl;
9
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLTableInfoImpl;
10
import java.io.File;
11
import java.io.FileNotFoundException;
12
import java.io.IOException;
13
import java.io.InputStream;
14
import java.io.Reader;
15
import java.nio.charset.Charset;
16
import java.util.ArrayList;
17
import java.util.List;
18
import java.util.Locale;
19
import javax.xml.parsers.SAXParser;
20
import javax.xml.parsers.SAXParserFactory;
21
import org.apache.commons.lang3.ArrayUtils;
22
import org.apache.commons.lang3.StringUtils;
23
import org.apache.tika.utils.CharsetUtils;
24
import org.cresques.cts.IProjection;
25
import org.gvsig.fmap.crs.CRSFactory;
26
import org.gvsig.fmap.dal.DALLocator;
27
import org.gvsig.fmap.dal.DataManager;
28
import org.gvsig.fmap.dal.DataTypes;
29
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor;
30
import org.gvsig.fmap.dal.feature.EditableFeatureType;
31
import org.gvsig.fmap.dal.feature.EditableForeingKey;
32
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;
33
import org.gvsig.tools.ToolsLocator;
34
import org.gvsig.tools.dynobject.DynField_v2;
35
import org.gvsig.tools.i18n.I18nManager;
36
import org.gvsig.tools.task.SimpleTaskStatus;
37
import org.gvsig.xml2db.lib.api.xmlinfo.XMLAttributeInfo;
38
import org.gvsig.xml2db.lib.api.xmlinfo.XMLInfo;
39
import org.gvsig.xml2db.lib.api.xmlinfo.XMLTableInfo;
40
import org.gvsig.xml2db.lib.impl.xmlinfo.XMLInfoImpl;
41
import org.slf4j.Logger;
42
import org.slf4j.LoggerFactory;
43
import org.xml.sax.Attributes;
44
import org.xml.sax.InputSource;
45
import org.xml.sax.Locator;
46
import org.xml.sax.SAXException;
47
import org.xml.sax.helpers.DefaultHandler;
48

    
49
/**
50
 *
51
 * @author jjdelcerro
52
 */
53
public class StructureExtractorImpl {
54

    
55
        private static final Logger LOGGER = LoggerFactory.getLogger(StructureExtractorImpl.class);
56

    
57
    @SuppressWarnings("UseSpecificCatch")
58
    private void extractTags(XMLInfoImpl xmlinfo, Reader reader, SimpleTaskStatus status) {
59
        if( reader == null ) {
60
            throw new IllegalArgumentException("reader is null");
61
        }
62
        try {
63
            final DataManager dataManager = DALLocator.getDataManager();
64
            I18nManager i18n = ToolsLocator.getI18nManager();
65
            status.message(i18n.getTranslation("_Reading_xml")+" 1/5");
66
            status.setRangeOfValues(0, xmlinfo.getCountLines());
67
            
68
            SAXParserFactory spf = SAXParserFactory.newInstance();
69
            spf.setNamespaceAware(true);
70
            SAXParser saxParser = spf.newSAXParser();
71
            InputSource is = new InputSource(reader);
72
            
73
            List<String> path = new ArrayList<>();
74
            
75
            saxParser.parse(is, new DefaultHandler() {
76
                private Locator locator;
77
                int size;
78
                int refreshInterval = 1;
79
                StringBuilder chars = new StringBuilder();
80
                
81
                @Override
82
                public void setDocumentLocator(Locator locator) {
83
                    this.locator = locator;
84
                }
85
                
86
                @Override
87
                public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
88
                    int line = this.locator.getLineNumber();
89
                    int column = this.locator.getColumnNumber()-2-localName.length();
90

    
91
                    if(line % refreshInterval == 0) {
92
                        status.setCurValue(line);
93
                    }
94
                    
95
                    if(line > 100){
96
                        refreshInterval = 100;
97
                    } else if(line > 1000){
98
                        refreshInterval = 1000;
99
                    } else if(line > 10000){
100
                        refreshInterval = 10000;
101
                    } else if(line > 100000){
102
                        refreshInterval = 100000;
103
                    }
104

    
105
                    String idvalue = dataManager.createUniqueID();
106
                    
107
                    path.add(localName);
108
                    String path_s = StringUtils.join(path, "/");
109
                    XMLAttributeInfoImpl info = xmlinfo.getTag(path_s);
110
                    if( info == null ) {
111
                        info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s);
112
                        xmlinfo.addTag(info);
113
                        
114
                    }
115
                    if( path.size()>1 ) {
116
                        List<String> parentpath = path.subList(0, path.size()-1);
117
                        String parentpath_s = StringUtils.join(parentpath, "/");
118
                        XMLAttributeInfoImpl parentinfo = xmlinfo.getTag(parentpath_s);
119
                        parentinfo.incrChildCount(localName);
120
                        parentinfo.setLastChildID(localName, idvalue);
121
                    }
122
                    
123
                    for (int i = 0; i < attributes.getLength(); i++) {
124
                        String name = attributes.getLocalName(i);                        
125
                        String value = attributes.getValue(i);
126
                        String idvalueChild = dataManager.createUniqueID();
127
                        XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name);
128
                        if( infoChild == null ) {
129
                            infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name);
130
                            xmlinfo.addTag(infoChild);
131
                        }
132
                        info.incrChildCount(infoChild.getName());
133
                        info.setLastChildID(infoChild.getName(), idvalueChild);
134
                        infoChild.addValue(value);
135
                        if( xmlinfo.getSrid() ==  null && StringUtils.containsIgnoreCase(infoChild.getName(), "srid") ) {
136
                            IProjection proj = getProjection(value);
137
                            if( proj!=null ) {
138
                                xmlinfo.setSrid(proj);
139
                            }
140
                        }
141
                    }
142
                    chars.setLength(0);
143
                }
144
                
145
                @Override
146
                public void endElement(String uri, String localName, String qName) throws SAXException {
147
                    int line = this.locator.getLineNumber();
148

    
149
//                    status.setCurValue(line);
150

    
151
                    String path_s = StringUtils.join(path, "/");
152
                    XMLAttributeInfoImpl info = xmlinfo.getTag(path_s);
153
                    if( info == null ) {
154
                        info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s);
155
                        xmlinfo.addTag(info);
156
                    }
157

    
158
                    XMLAttributeInfoImpl parentinfo = null;
159
                    if( path.size()>1 ) {
160
                        List<String> parentpath = path.subList(0, path.size()-1);
161
                        String parentpath_s = StringUtils.join(parentpath, "/");
162
                        parentinfo = xmlinfo.getTag(parentpath_s);
163
                    }
164
                    
165
                    if( info.hasChilds() || (parentinfo != null && (parentinfo.getChildsCount(localName)>1))) {
166
                        String value = this.chars.toString();
167
                        if( StringUtils.isNotBlank(value) ) {
168
                            String name = info.getName()+"$v";                        
169
                            String idvalueChild = dataManager.createUniqueID();
170
                            XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name);
171
                            if( infoChild == null ) {
172
                                infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name);
173
                                xmlinfo.addTag(infoChild);
174
                            }
175
                            info.incrChildCount(infoChild.getName());
176
                            info.setLastChildID(infoChild.getName(), idvalueChild);
177
                            infoChild.addValue(value);
178
                        }
179
                    } else {
180
                        String value = this.chars.toString();
181
                        info.addValue(value);
182
                        if( StringUtils.containsIgnoreCase(info.getName(), "srid") ) {
183
                            IProjection proj = getProjection(value);
184
                            if( proj!=null ) {
185
                                xmlinfo.setSrid(proj);
186
                            }
187
                        }
188
                    }
189
                    info.consolidateChildCounters();
190
//                    if( StringUtils.equalsIgnoreCase("LUGAR_CIRCULABA", info.getName()) ) {
191
//                        System.out.println("Oh!");
192
//                    }
193
                    
194
                    path.remove(path.size()-1);
195
                    chars.setLength(0);
196
                }
197
                
198
                @Override
199
                public void characters(char[] ch, int start, int length) throws SAXException {
200
                    int line = this.locator.getLineNumber();
201

    
202
//                    status.setCurValue(line);
203
                    this.chars.append(ch, start, length);
204
                }
205
                
206
                
207
            });
208
        } catch (Exception ex) {
209
            throw new RuntimeException("Can't extract tags.", ex);
210
        }
211
    }
212
    
213
    private IProjection getProjection(String value) {
214
        if( StringUtils.isBlank(value) ) {
215
            return null;
216
        }
217
        IProjection proj = null;
218
        try {
219
            proj = CRSFactory.getCRS(value);
220
        } catch(Throwable t) {
221
            
222
        }
223
        if( proj != null ) {
224
            return proj;
225
        }
226
        try {
227
            proj = CRSFactory.getCRS("EPSG:"+value);
228
        } catch(Throwable t) {
229
            
230
        }
231
        return proj;
232
    }
233
    
234
    public XMLInfo extractStructure(File xml, Charset charset, IProjection projection, Locale locale, SimpleTaskStatus status) throws FileNotFoundException, IOException {
235
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
236
        xmlinfo.setLocale(locale);
237
        xmlinfo.setSrid(projection);
238
        long count = Xml2dbCommons.countLines(xml, charset, status);
239
        xmlinfo.setCountLines(count);
240
        InputSource is = Xml2dbCommons.openReader(xml,charset);
241
        return extractStructure(is, xmlinfo, status);
242
    }
243
    
244
    public XMLInfo extractStructure(InputStream xml, Charset charset, IProjection projection, Locale locale, SimpleTaskStatus status) throws IOException  {
245
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
246
        xmlinfo.setLocale(locale);
247
        xmlinfo.setSrid(projection);
248
//        long count = Xml2dbCommons.countLines(xml, charset, status);
249
        xmlinfo.setCountLines(-1);
250
        InputSource is = Xml2dbCommons.openReader(xml, charset);
251
        return extractStructure(is, xmlinfo, status);
252
    }
253
    
254
    public XMLInfo extractStructure(Reader reader, IProjection projection, Locale locale, SimpleTaskStatus status) {
255
        XMLInfoImpl xmlinfo = new XMLInfoImpl();
256
        xmlinfo.setLocale(locale);
257
        xmlinfo.setSrid(projection);
258
        InputSource is = new InputSource(reader);
259
        return extractStructure(is, xmlinfo, status);
260
    }
261
    
262
    public XMLInfo extractStructure(InputSource is, XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
263
        
264
        if( xmlinfo.getCharset()==null ) {
265
            xmlinfo.setCharset(CharsetUtils.forName(is.getEncoding()));
266
        }
267
        
268
        extractTags(xmlinfo, is.getCharacterStream(), status);
269
        
270
        buildTablesFromTags(xmlinfo, status);
271

    
272
        removeDuplicateTableNames(xmlinfo, status);
273
        
274
        createPrimaryKeyAndForeignKeys(xmlinfo, status);
275
        
276
        createFeatureTypes(xmlinfo, status);
277

    
278
        return xmlinfo;
279
    }
280
    
281
    private String plural(String s) {
282
        int l = s.length();
283
        char ch = s.substring(l-1,l).toLowerCase().charAt(0);
284
        if( ch=='a' || ch=='e' || ch=='i' || ch=='o' || ch=='u' ) {
285
            return s+"s";
286
        }
287
        return s+"es";
288
    }
289

    
290
    private void buildTablesFromTags(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
291
        I18nManager i18n = ToolsLocator.getI18nManager();
292
        status.message(i18n.getTranslation("_Identifying_tables")+" 2/5");
293
        status.setRangeOfValues(0, xmlinfo.getTagsPaths().size());
294
        
295
        for (String tagPath : xmlinfo.getTagsPaths()) {
296
            XMLAttributeInfoImpl tag1Info = xmlinfo.getTagInfo(tagPath);
297
//            System.out.println(tag1Info.getPath());
298
            XMLTableInfoImpl tableInfo = new XMLTableInfoImpl(tagPath, tag1Info);
299
            for (String tagPath2 : xmlinfo.getTagsPaths()) {
300
                if( tagPath.equals(tagPath2) ) {
301
                    continue;
302
                }
303
                if( tagPath2.startsWith(tagPath+"/") ) {
304
                    String fieldName = tagPath2.substring(tagPath.length()+1);
305
                    if( !fieldName.contains("/") ) {
306
                        XMLAttributeInfoImpl info = xmlinfo.getTag(tagPath2);
307
                        tableInfo.add(info);
308
                    }
309
                }
310
            }
311
            if( !tableInfo.isEmpty() ) {
312
                xmlinfo.addTable(tableInfo);
313
            }
314
            status.incrementCurrentValue();
315
        }
316
    }
317
    
318
    private void removeDuplicateTableNames(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
319
        I18nManager i18n = ToolsLocator.getI18nManager();
320
        status.message(i18n.getTranslation("_Fixing_table_names")+" 3/5");
321
        status.setRangeOfValues(0, xmlinfo.size());
322

    
323
        // Tratamos de corregir nombres duplicados en las tablas.
324
        for (XMLTableInfo tableInfo01 : xmlinfo) {
325
            XMLTableInfoImpl tableInfo1 = (XMLTableInfoImpl) tableInfo01;
326
            
327
            status.message(i18n.getTranslation("_Fixing_table_names")+" 3/5 ("+tableInfo1.getName()+")");
328

    
329
            boolean renombrar = false;
330
            for (XMLTableInfo tableInfo02 : xmlinfo) {
331
                XMLTableInfoImpl tableInfo2 = (XMLTableInfoImpl) tableInfo02;
332
                if( tableInfo1 == tableInfo2 ) {
333
                    continue;
334
                }
335
                if( StringUtils.equalsIgnoreCase(tableInfo1.getName(), tableInfo2.getName()) ) {
336
                    // FIXME: solo usa los dos ultimos nombres, podrian coincidir igualmente!!
337
                    String[] path_ss = tableInfo2.getPath().split("/");
338
                    int l = path_ss.length;
339
                    String name = path_ss[l-1]+"_"+path_ss[l-2];
340
                    tableInfo2.rename(name);
341
                    renombrar = true;
342
//                    System.out.println("###: Tablas con el mismo nombre, renombrado a "+name);
343
//                    System.out.println("###: "+tableInfo1.getPath());
344
//                    System.out.println("###: "+tableInfo2.getPath());
345
                }
346
            }
347
            if( renombrar ) {
348
                // FIXME: solo usa los dos ultimos nombres, podrian coincidir igualmente!!
349
                String[] path_ss = tableInfo1.getPath().split("/");
350
                int l = path_ss.length;
351
                String name = path_ss[l-1]+"_"+path_ss[l-2];
352
                tableInfo1.rename(name);
353
            }
354
            status.incrementCurrentValue();
355
        }
356
    }
357
    
358
    private void createPrimaryKeyAndForeignKeys(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
359
        I18nManager i18n = ToolsLocator.getI18nManager();
360
        status.message(i18n.getTranslation("_Identifying_foreign_keys")+" 4/5");
361
        status.setRangeOfValues(0, xmlinfo.size());
362

    
363
        for (XMLTableInfo tableInfo0 : xmlinfo) {
364
            XMLTableInfoImpl tableInfo = (XMLTableInfoImpl) tableInfo0;
365
            
366
            status.message(i18n.getTranslation("_Identifying_foreign_keys")+" 4/5 ("+tableInfo.getName()+")");
367

    
368
            String[] fieldkeys = tableInfo.getPath().split("/");
369
            for (int i = 0; i < fieldkeys.length; i++) {
370
                String fieldkey = fieldkeys[i];
371
                if( i==fieldkeys.length-1 ) {
372
                    tableInfo.add(new XMLAttributeInfoImpl(xmlinfo.getLocale(), "$ID_"+tableInfo.getName(), DataTypes.INT).setPk(true));
373
                } else { 
374
                    // TODO es una relacion 1:1
375
                    XMLTableInfo tableInfo2 = xmlinfo.getTableByPath(StringUtils.join(ArrayUtils.subarray(fieldkeys, 0, i+1),"/"));
376
                    tableInfo.add(new XMLAttributeInfoImpl(xmlinfo.getLocale(), "$ID_"+tableInfo2.getName(), DataTypes.INT)
377
                            .setFk(true)
378
                            .setFkCodeName("$ID_"+fieldkey)
379
                            .setFkTableName(fieldkey)
380
                    );
381
                }
382
            }
383

    
384
            for (XMLAttributeInfo field0 : tableInfo) {
385
                XMLAttributeInfoImpl field = (XMLAttributeInfoImpl) field0;
386
                if( xmlinfo.existsTableByPath(tableInfo.getPath()+"/"+field.getName()) ) {
387
                    XMLTableInfoImpl fktableInfo = xmlinfo.getTableByPath(tableInfo.getPath()+"/"+field.getName());
388
                    int count = tableInfo.getMaxCountChild(field.getName());
389
                    if( count<1 ) {
390
                        // Do nothing
391
                    } else if( count>1 ) {
392
                        // TODO es una relacion 1:n
393
                        field.setAggregate(true);
394
                        field.setFkCodeName("$ID_"+tableInfo.getName());
395
                        field.setFkTableName(fktableInfo.getName());
396
                        field.setSize(45);
397
                        field.setType(DataTypes.LIST);
398
                    } else {
399
                        // TODO es una relacion 1:1
400
                        field.setFk(true);
401
                        field.setFkCodeName("$ID_"+fktableInfo.getName());
402
                        field.setFkTableName(fktableInfo.getName());
403
                        field.setSize(0);
404
                        field.setType(DataTypes.INT);
405
                    }
406
                }
407
            }            
408
            tableInfo.sort();
409
//            System.out.println(tableInfo);
410
            status.incrementCurrentValue();
411
        }
412
    }
413
    
414
    private void createFeatureTypes(XMLInfoImpl xmlinfo, SimpleTaskStatus status) {
415
        I18nManager i18n = ToolsLocator.getI18nManager();
416
        status.message(i18n.getTranslation("_Creating_table_definitions")+" 5/5");
417
        status.setRangeOfValues(0, xmlinfo.size());
418

    
419
        DataManager dataManager = DALLocator.getDataManager();
420

    
421
        for (XMLTableInfo tableInfo0 : xmlinfo) {
422
            XMLTableInfoImpl tableInfo = (XMLTableInfoImpl) tableInfo0;
423

    
424
            status.message(i18n.getTranslation("_Creating_table_definitions")+" 5/5 ("+tableInfo.getName()+")");
425
            
426
            EditableFeatureType ft = dataManager.createFeatureType();
427
            ft.setLabel(tableInfo.getName());
428
            ft.getTags().set("xml2db.tablename", tableInfo.getName());
429
            ft.getTags().set("vcsgis.storename", tableInfo.getName());
430
            ft.getTags().set("xml2db.path", tableInfo.getPath());
431
            for (XMLAttributeInfo attrinfo0 : tableInfo) {
432
                XMLAttributeInfoImpl attrinfo = (XMLAttributeInfoImpl) attrinfo0;
433
                if( ft.get(attrinfo.getName())!=null ) {
434
                    LOGGER.warn("Duplicated attribute '"+attrinfo.getName()+"' in '"+tableInfo.getName()+"'.");
435
                }
436
                if(  attrinfo.isAggregate() ) {
437
                    continue;
438
                }
439
                EditableFeatureAttributeDescriptor attrdesc;
440
                if( attrinfo.getName().startsWith("$ID") ) {
441
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
442
                    attrdesc.setLabel("Id. "+attrinfo.getName().substring(4));
443
                    attrdesc.setSize(attrinfo.getSize());
444
                } else if( attrinfo.getName().endsWith("$v") ) {
445
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
446
                    attrdesc.setLabel(StringUtils.left(attrinfo.getName(),attrinfo.getName().length()-2));
447
                    attrdesc.setSize(attrinfo.getSize());
448
                } else {
449
                    attrdesc = ft.add(attrinfo.getName(), attrinfo.getType());
450
                    attrdesc.setSize(attrinfo.getSize());
451
                    attrdesc.setPrecision(attrinfo.getPrecision());
452
                    attrdesc.setScale(attrinfo.getScale());
453
                    if( attrdesc.getType()==DataTypes.GEOMETRY ) {
454
                        attrdesc.setGeometryType(attrinfo.getGeometryType());
455
                        attrdesc.setSRS(xmlinfo.getSrid());
456
                    }
457
                }
458
                attrdesc.setLocale(xmlinfo.getLocale());
459
                attrdesc.setIsPrimaryKey(attrinfo.isPk());
460
                if( attrinfo.isFk() ) {
461
                    EditableForeingKey fk = attrdesc.getForeingKey();
462
                    fk.setForeingKey(true);
463
                    fk.setTableName(attrinfo.getFkTableName());
464
                    fk.setCodeName(attrinfo.getFkCodeName());
465
                    fk.setClosedList(false);
466
                    fk.setLabelFormula("\""+attrinfo.getFkCodeName()+"\"");
467
                    attrdesc.setRelationType(DynField_v2.RELATION_TYPE_COLLABORATION);
468
                    attrdesc.setIsIndexed(true);
469
                }
470
            }
471
            FeatureAttributeDescriptor pk = ft.getPrimaryKey()[0];
472
            for (XMLAttributeInfo attrinfo0 : tableInfo) {
473
                XMLAttributeInfoImpl attrinfo = (XMLAttributeInfoImpl) attrinfo0;
474
                if(  attrinfo.isAggregate() ) {
475
                    EditableFeatureAttributeDescriptor attrdesc = ft.add("$List_"+attrinfo.getName(), attrinfo.getType());
476
                    attrdesc.setLabel(plural(attrinfo.getName()));
477
                    attrdesc.getTags().set("dal.relatedfeatures.table", attrinfo.getFkTableName());
478
                    attrdesc.getTags().set("dal.relatedfeatures.unique.field.name", attrinfo.getFkCodeName());
479
                    attrdesc.getTags().set("dynform.label.empty", true);
480
                    attrdesc.getTags().set("dynform.resizeWeight", 100);
481
                    attrdesc.setFeatureAttributeEmulator(
482
                        "SELECT * FROM \""+attrinfo.getFkTableName() + "\" WHERE ( (\""+pk.getName()+"\") = (\""+attrinfo.getFkTableName() + "\".\""+pk.getName()+"\") )"                    
483
                    );
484
                    attrdesc.setRelationType(DynField_v2.RELATION_TYPE_AGGREGATE);
485
                }
486
            }
487
            tableInfo.setFeatureType(ft);
488
            status.incrementCurrentValue();
489
        }
490
    }
491
}