Revision 47634
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/XMLFileAsList.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license header, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.csv; |
|
7 |
|
|
8 |
import java.io.BufferedReader; |
|
9 |
import java.io.Closeable; |
|
10 |
import java.io.File; |
|
11 |
import java.io.IOException; |
|
12 |
import java.nio.ByteBuffer; |
|
13 |
import java.nio.CharBuffer; |
|
14 |
import java.nio.charset.Charset; |
|
15 |
import java.nio.charset.StandardCharsets; |
|
16 |
import java.util.AbstractList; |
|
17 |
import java.util.ArrayList; |
|
18 |
import java.util.HashMap; |
|
19 |
import java.util.List; |
|
20 |
import java.util.Map; |
|
21 |
import java.util.Objects; |
|
22 |
import javax.xml.parsers.SAXParser; |
|
23 |
import javax.xml.parsers.SAXParserFactory; |
|
24 |
import org.apache.commons.io.FilenameUtils; |
|
25 |
import org.apache.commons.io.IOUtils; |
|
26 |
import org.apache.commons.lang3.StringUtils; |
|
27 |
import org.apache.commons.lang3.tuple.Pair; |
|
28 |
import org.gvsig.fmap.dal.store.csv.RecordsFile.Record; |
|
29 |
import org.gvsig.fmap.dal.store.csv.RecordsFile.RecordType; |
|
30 |
import static org.gvsig.fmap.dal.store.csv.RecordsFile.RecordTypeBuilder.recordTypeBuilder; |
|
31 |
import org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileReader; |
|
32 |
import org.gvsig.tools.util.GetItemWithSize64; |
|
33 |
import org.gvsig.tools.util.ListBuilder; |
|
34 |
import org.xml.sax.Attributes; |
|
35 |
import org.xml.sax.InputSource; |
|
36 |
import org.xml.sax.Locator; |
|
37 |
import org.xml.sax.SAXException; |
|
38 |
import org.xml.sax.helpers.DefaultHandler; |
|
39 |
|
|
40 |
/** |
|
41 |
* |
|
42 |
* @author jjdelcerro |
|
43 |
*/ |
|
44 |
@SuppressWarnings("UseSpecificCatch") |
|
45 |
public class XMLFileAsList |
|
46 |
extends AbstractList<List<String>> |
|
47 |
implements Closeable, GetItemWithSize64<List<String>> { |
|
48 |
|
|
49 |
private static final int IDXFIELD_LINE = 0; |
|
50 |
private static final int IDXFIELD_COLUMN = 1; |
|
51 |
private static final int IDXFIELD_LINEPOS = 2; |
|
52 |
private static final int IDXFIELD_RECORDPOS = 3; |
|
53 |
|
|
54 |
private final RandomAccessFileReader reader; |
|
55 |
private final String recordPath; |
|
56 |
private RecordsFile index; |
|
57 |
private List<String> fieldPaths; |
|
58 |
|
|
59 |
|
|
60 |
// ruta recordnum ruta recordnum |
|
61 |
// public XMLFileAsList(File text, Charset charset, List<Pair<String,Integer>> recordPath, List<Pair<String,Integer>> fieldPaths) throws IOException { |
|
62 |
public XMLFileAsList(File text, Charset charset, String recordPath, List<String> fieldPaths) throws IOException { |
|
63 |
this.reader = new RandomAccessFileReader(text, charset); |
|
64 |
this.index = null; |
|
65 |
this.recordPath = recordPath; |
|
66 |
} |
|
67 |
|
|
68 |
public XMLFileAsList(File text, File index, Charset charset, String recordPath, List<String> fieldPaths) throws IOException { |
|
69 |
this.reader = new RandomAccessFileReader(text, charset); |
|
70 |
if (index.exists()) { |
|
71 |
// TODO: Force to create index if text newer than index |
|
72 |
this.index = new RecordsFileImpl(index); |
|
73 |
} else { |
|
74 |
this.createIndex(index); |
|
75 |
} |
|
76 |
this.recordPath = recordPath; |
|
77 |
} |
|
78 |
|
|
79 |
public XMLFileAsList(RandomAccessFileReader reader, RecordsFile index, String recordPath, List<String> fieldPaths) throws IOException { |
|
80 |
this.reader = reader; |
|
81 |
this.index = index; |
|
82 |
this.recordPath = recordPath; |
|
83 |
} |
|
84 |
|
|
85 |
@Override |
|
86 |
public void close() { |
|
87 |
IOUtils.closeQuietly(this.reader); |
|
88 |
IOUtils.closeQuietly(this.index); |
|
89 |
} |
|
90 |
|
|
91 |
private List<String> getRecord(long position, List<String> fieldPaths) { |
|
92 |
class StopParserSAXException extends SAXException { |
|
93 |
} |
|
94 |
|
|
95 |
class ParseRecordsHandler extends DefaultHandler { |
|
96 |
|
|
97 |
Locator locator; |
|
98 |
List<String> path = new ArrayList<>(); |
|
99 |
Map<String, String> record = new HashMap<>(); |
|
100 |
StringBuilder value = new StringBuilder(); |
|
101 |
List<String> values = new ArrayList<>(); |
|
102 |
|
|
103 |
@Override |
|
104 |
public void setDocumentLocator(Locator locator) { |
|
105 |
this.locator = locator; |
|
106 |
} |
|
107 |
|
|
108 |
@Override |
|
109 |
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { |
|
110 |
path.add(localName); |
|
111 |
String path_s = StringUtils.join(path, "/"); |
|
112 |
if (StringUtils.equalsIgnoreCase(path_s, recordPath)) { |
|
113 |
return; |
|
114 |
} |
|
115 |
this.value.setLength(0); |
|
116 |
// this.value.trimToSize(); |
|
117 |
} |
|
118 |
|
|
119 |
@Override |
|
120 |
public void characters(char[] ch, int start, int length) throws SAXException { |
|
121 |
value.append(new String(ch, start, length)); |
|
122 |
} |
|
123 |
|
|
124 |
@Override |
|
125 |
public void endElement(String uri, String localName, String qName) throws SAXException { |
|
126 |
String path_s = StringUtils.join(path, "/"); |
|
127 |
|
|
128 |
if (StringUtils.equalsIgnoreCase(path_s, recordPath)) { |
|
129 |
for (String fieldPath : fieldPaths) { |
|
130 |
values.add(record.get(fieldPath)); |
|
131 |
} |
|
132 |
throw new StopParserSAXException(); |
|
133 |
} else { |
|
134 |
for (String fieldPath : fieldPaths) { |
|
135 |
if (StringUtils.equalsIgnoreCase(path_s, fieldPath)) { |
|
136 |
record.put(fieldPath, this.value.toString()); |
|
137 |
// No break to handle repeated columns |
|
138 |
} |
|
139 |
} |
|
140 |
} |
|
141 |
|
|
142 |
path.remove(path.size() - 1); |
|
143 |
} |
|
144 |
} |
|
145 |
|
|
146 |
ParseRecordsHandler handler = null; |
|
147 |
try { |
|
148 |
Record record = this.index.get64(position); |
|
149 |
long recordPosition = record.getLong(IDXFIELD_RECORDPOS); |
|
150 |
this.reader.seek(recordPosition); |
|
151 |
BufferedReader breader = new BufferedReader(this.reader, 1024 * 8); |
|
152 |
|
|
153 |
InputSource is = new InputSource(breader); |
|
154 |
SAXParserFactory spf = SAXParserFactory.newInstance(); |
|
155 |
spf.setNamespaceAware(true); |
|
156 |
SAXParser saxParser = spf.newSAXParser(); |
|
157 |
handler = new ParseRecordsHandler(); |
|
158 |
|
|
159 |
saxParser.parse(is, handler); |
|
160 |
} catch (StopParserSAXException ex) { |
|
161 |
if (handler != null) { |
|
162 |
return handler.values; |
|
163 |
} |
|
164 |
} catch (Exception ex) { |
|
165 |
throw new RuntimeException("Can't parse record " + position, ex); |
|
166 |
} |
|
167 |
return null; // ????? |
|
168 |
} |
|
169 |
|
|
170 |
@Override |
|
171 |
public List<String> get(int index) { |
|
172 |
return this.getRecord(index, fieldPaths); |
|
173 |
} |
|
174 |
|
|
175 |
@Override |
|
176 |
public List<String> get64(long position) { |
|
177 |
return this.getRecord(position, fieldPaths); |
|
178 |
} |
|
179 |
|
|
180 |
@Override |
|
181 |
public int size() { |
|
182 |
return this.index.size(); |
|
183 |
} |
|
184 |
|
|
185 |
@Override |
|
186 |
public long size64() { |
|
187 |
return this.index.size64(); |
|
188 |
} |
|
189 |
|
|
190 |
final public void createIndex(File indexFile) throws IOException { |
|
191 |
try { |
|
192 |
// 1. Creamos el indice vacio |
|
193 |
RecordType recordType = recordTypeBuilder() |
|
194 |
.addLong() // IDXFIELD_LINE |
|
195 |
.addLong() // IDXFIELD_COLUMN |
|
196 |
.addLong() // IDXFIELD_LINEPOS |
|
197 |
.addLong() // IDXFIELD_RECORDPOS |
|
198 |
.build(); |
|
199 |
Record record = recordType.createRecord(); |
|
200 |
final RecordsFile theIndex = new RecordsFileImpl(); |
|
201 |
theIndex.create(indexFile, recordType, 0); |
|
202 |
|
|
203 |
// 2. Rellenamos los campos numero de linea y columna de cada registro |
|
204 |
// en el indice. |
|
205 |
// Para ello nos recorremos el XML y vamos escribiendo en el indice. |
|
206 |
this.reader.rewind(); |
|
207 |
InputSource is = new InputSource(reader); |
|
208 |
SAXParserFactory spf = SAXParserFactory.newInstance(); |
|
209 |
spf.setNamespaceAware(true); |
|
210 |
SAXParser saxParser = spf.newSAXParser(); |
|
211 |
|
|
212 |
List<String> path = new ArrayList<>(); |
|
213 |
// TODO: Aqui habria que quedarse con la primera aparicion de |
|
214 |
// los campos solicitados que no pertenezcan al registro principal. |
|
215 |
// Probablemente habria que guardar esos valores en disco junto al |
|
216 |
// fichero de indice. |
|
217 |
saxParser.parse(is, new DefaultHandler() { |
|
218 |
Locator locator; |
|
219 |
|
|
220 |
@Override |
|
221 |
public void setDocumentLocator(Locator locator) { |
|
222 |
this.locator = locator; |
|
223 |
} |
|
224 |
|
|
225 |
@Override |
|
226 |
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { |
|
227 |
path.add(localName); |
|
228 |
String path_s = StringUtils.join(path, "/"); |
|
229 |
|
|
230 |
if (StringUtils.equalsIgnoreCase(path_s, recordPath)) { |
|
231 |
record.setLong(IDXFIELD_LINE, locator.getLineNumber() - 1); |
|
232 |
record.setLong(IDXFIELD_COLUMN, locator.getColumnNumber() - 1); |
|
233 |
record.setLong(IDXFIELD_LINEPOS, 0); |
|
234 |
record.setLong(IDXFIELD_RECORDPOS, 0); |
|
235 |
theIndex.add(record); |
|
236 |
} |
|
237 |
} |
|
238 |
|
|
239 |
@Override |
|
240 |
public void endElement(String uri, String localName, String qName) throws SAXException { |
|
241 |
path.remove(path.size() - 1); |
|
242 |
} |
|
243 |
}); |
|
244 |
|
|
245 |
// 3. Nos recorremos el indice y calculamos la posicion de la linea |
|
246 |
// para cada registro. |
|
247 |
this.reader.rewind(); |
|
248 |
PositionCalculator positionCalculator = new PositionCalculator(reader); |
|
249 |
|
|
250 |
for (int i = 0; i < theIndex.size(); i++) { |
|
251 |
Record r = theIndex.get(i); |
|
252 |
positionCalculator.next(r.getLong(IDXFIELD_LINE), r.getLong(IDXFIELD_COLUMN)); |
|
253 |
r.setLong(IDXFIELD_LINEPOS, positionCalculator.getLinePosition()); |
|
254 |
r.setLong(IDXFIELD_RECORDPOS, positionCalculator.getColumnPosition()); |
|
255 |
theIndex.set(i, record); |
|
256 |
} |
|
257 |
this.index = new RecordsFileImpl(indexFile); |
|
258 |
} catch (Exception ex) { |
|
259 |
throw new IOException("Can't create index " + Objects.toString(indexFile), ex); |
|
260 |
} |
|
261 |
} |
|
262 |
|
|
263 |
private static class CharacterSize { |
|
264 |
|
|
265 |
private final Charset charset; |
|
266 |
private final CharBuffer charBuffer; |
|
267 |
|
|
268 |
public CharacterSize(Charset charset) { |
|
269 |
this.charset = charset; |
|
270 |
this.charBuffer = CharBuffer.allocate(1); |
|
271 |
} |
|
272 |
|
|
273 |
public int size(char ch) { |
|
274 |
this.charBuffer.put(0, ch); |
|
275 |
this.charBuffer.position(0); |
|
276 |
ByteBuffer buffer = this.charset.encode(this.charBuffer); |
|
277 |
return buffer.limit(); |
|
278 |
} |
|
279 |
|
|
280 |
public int size(char[] cbuf, int off, int len) { |
|
281 |
CharBuffer cb = CharBuffer.wrap(cbuf, off, len); |
|
282 |
ByteBuffer byteBuffer = this.charset.encode(cb); |
|
283 |
return byteBuffer.limit(); |
|
284 |
} |
|
285 |
|
|
286 |
public int size(char[] cbuf) { |
|
287 |
CharBuffer cb = CharBuffer.wrap(cbuf, 0, cbuf.length); |
|
288 |
ByteBuffer byteBuffer = this.charset.encode(cb); |
|
289 |
return byteBuffer.limit(); |
|
290 |
} |
|
291 |
|
|
292 |
public int size(String s) { |
|
293 |
CharBuffer cb = CharBuffer.wrap(s); |
|
294 |
ByteBuffer byteBuffer = this.charset.encode(cb); |
|
295 |
return byteBuffer.limit(); |
|
296 |
} |
|
297 |
} |
|
298 |
|
|
299 |
private static class PositionCalculator { |
|
300 |
|
|
301 |
private final RandomAccessFileReader reader; |
|
302 |
private final BufferedReader breader; |
|
303 |
private long currentPosition; |
|
304 |
private long currentColumn; |
|
305 |
private long currentLine; |
|
306 |
private long currentColumnPosition; |
|
307 |
private long currentLinePosition; |
|
308 |
|
|
309 |
private final char[] ch; |
|
310 |
private final CharacterSize characterSize; |
|
311 |
|
|
312 |
public PositionCalculator(RandomAccessFileReader reader) { |
|
313 |
this.reader = reader; |
|
314 |
this.breader = new BufferedReader(this.reader, 1024 * 8); |
|
315 |
this.currentPosition = this.reader.getCurrentPosition(); |
|
316 |
this.currentLine = 0; |
|
317 |
this.currentColumn = 0; |
|
318 |
this.ch = new char[1]; |
|
319 |
this.characterSize = new CharacterSize(this.reader.getCharset()); |
|
320 |
} |
|
321 |
|
|
322 |
public boolean next(long line, long column) throws IOException { |
|
323 |
while (this.currentLine < line) { |
|
324 |
if (breader.read(this.ch, 0, 1) < 1) { |
|
325 |
return false; |
|
326 |
} |
|
327 |
char c = ch[0]; |
|
328 |
this.currentPosition += characterSize.size(c); |
|
329 |
if (c == '\n') { |
|
330 |
this.currentLine++; |
|
331 |
this.currentColumn = 0; |
|
332 |
} |
|
333 |
} |
|
334 |
this.currentLinePosition = this.currentPosition; |
|
335 |
while (this.currentColumn < column) { |
|
336 |
if (breader.read(this.ch, 0, 1) < 1) { |
|
337 |
return false; |
|
338 |
} |
|
339 |
char c = ch[0]; |
|
340 |
this.currentPosition += characterSize.size(c); |
|
341 |
if (c == '\n') { |
|
342 |
// Uff, esto seria un error, ya que la linea corriente no |
|
343 |
// tiene tantas columnas. |
|
344 |
throw new IOException("Illegal column number " + column + " for line " + line); |
|
345 |
} |
|
346 |
this.currentColumn++; |
|
347 |
} |
|
348 |
this.currentColumnPosition = this.currentPosition; |
|
349 |
return true; |
|
350 |
} |
|
351 |
|
|
352 |
public long getLinePosition() { |
|
353 |
return this.currentLinePosition; |
|
354 |
} |
|
355 |
|
|
356 |
public long getColumnPosition() { |
|
357 |
return this.currentColumnPosition; |
|
358 |
} |
|
359 |
|
|
360 |
} |
|
361 |
|
|
362 |
public static void main(String[] args) throws Exception { |
|
363 |
final String XMLFILE1 = "/home/jjdelcerro/datos/geodata/vector/gml/navarra.gml"; |
|
364 |
final String XMLFILE2 = "/home/jjdelcerro/datos/geodata/vector/gml/Municipis/Municipis.gml"; |
|
365 |
|
|
366 |
String gmlfile = XMLFILE1; |
|
367 |
XMLFileAsList gml = new XMLFileAsList( |
|
368 |
new File(gmlfile), |
|
369 |
new File(FilenameUtils.removeExtension(gmlfile)+".gmlidx"), |
|
370 |
StandardCharsets.UTF_8, |
|
371 |
"FeatureCollection/featureMember", |
|
372 |
ListBuilder.create( |
|
373 |
"FEATURE/fid", |
|
374 |
"FEATURE/MERINDAD", |
|
375 |
"FEATURE/Cnt_MERIND", |
|
376 |
"FEATURE/Nombre" |
|
377 |
) |
|
378 |
); |
|
379 |
System.out.println("File: "+gmlfile); |
|
380 |
for (int i = 0; i < gml.size(); i++) { |
|
381 |
List<String> item = gml.get(i); |
|
382 |
System.out.println(item); |
|
383 |
} |
|
384 |
} |
|
385 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVUtils.java | ||
---|---|---|
46 | 46 |
import static org.gvsig.fmap.dal.store.csv.CSVStoreProvider.NAME; |
47 | 47 |
import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReaderSuperCSV; |
48 | 48 |
import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader; |
49 |
import org.gvsig.fmap.dal.store.csv.simplereaders.GMLReader; |
|
49 | 50 |
import org.gvsig.fmap.dal.store.csv.simplereaders.JSonReader; |
50 | 51 |
import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader; |
51 | 52 |
import org.gvsig.fmap.geom.Geometry; |
... | ... | |
164 | 165 |
String filename = CSVStoreParameters.getFileName(parameters); |
165 | 166 |
if (FilenameUtils.isExtension(filename, "json")){ |
166 | 167 |
reader= new JSonReader(in,parameters); |
168 |
} else if (FilenameUtils.isExtension(filename, "gml")){ |
|
169 |
reader= new GMLReader(in,parameters); |
|
167 | 170 |
} else if (CSVStoreParameters.getRawFieldsDefinition(parameters) != null) { |
168 | 171 |
reader = new FixedLenReader(in, parameters); |
169 | 172 |
} else { |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/simplereaders/GMLReader.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license theHeader, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.csv.simplereaders; |
|
7 |
|
|
8 |
import java.io.File; |
|
9 |
import java.io.IOException; |
|
10 |
import java.io.Reader; |
|
11 |
import java.nio.charset.Charset; |
|
12 |
import java.util.List; |
|
13 |
import org.apache.commons.io.FilenameUtils; |
|
14 |
import org.gvsig.fmap.dal.store.csv.CSVStoreParameters; |
|
15 |
import org.gvsig.fmap.dal.store.csv.xml.GfsFile; |
|
16 |
import org.gvsig.fmap.dal.store.csv.xml.XMLFileAsList; |
|
17 |
import org.slf4j.Logger; |
|
18 |
import org.slf4j.LoggerFactory; |
|
19 |
|
|
20 |
/** |
|
21 |
* |
|
22 |
* @author gvSIG Team |
|
23 |
*/ |
|
24 |
public class GMLReader extends AbstractSimpleReader { |
|
25 |
|
|
26 |
private static final Logger LOGGER = LoggerFactory.getLogger(GMLReader.class); |
|
27 |
|
|
28 |
private XMLFileAsList gml; |
|
29 |
private int columns; |
|
30 |
private String[] header; |
|
31 |
private int currentElement = 0; |
|
32 |
private final CSVStoreParameters parameters; |
|
33 |
private GfsFile gfs; |
|
34 |
|
|
35 |
public GMLReader(Reader reader, CSVStoreParameters csvParameters) throws IOException { |
|
36 |
File gmlFile = csvParameters.getFile(); |
|
37 |
File gmlIdx = new File(FilenameUtils.removeExtension(gmlFile.getName())+".gmlidx"); |
|
38 |
File gfsFile = new File(FilenameUtils.removeExtension(gmlFile.getName())+".gfs"); |
|
39 |
gfs = new GfsFile(); |
|
40 |
if(gmlIdx.exists()){ |
|
41 |
//TODO: Comparar fechas del gfs y gml |
|
42 |
gfs.load(gfsFile); |
|
43 |
} else { |
|
44 |
gfs.fetch(gmlFile); |
|
45 |
gfs.save(gfsFile); |
|
46 |
} |
|
47 |
|
|
48 |
this.gml = new XMLFileAsList( |
|
49 |
gmlFile, |
|
50 |
gmlIdx, |
|
51 |
Charset.forName(CSVStoreParameters.getCharset(csvParameters)), |
|
52 |
gfs.getBaseElementPath(), |
|
53 |
gfs.getGeometryElementPaths(), |
|
54 |
gfs.getPropertiesPaths() |
|
55 |
); |
|
56 |
this.columns = -1; |
|
57 |
this.header = null; |
|
58 |
this.parameters = csvParameters; |
|
59 |
} |
|
60 |
|
|
61 |
@Override |
|
62 |
public String[] getHeader() throws IOException { |
|
63 |
if (this.header == null) { |
|
64 |
String[] theHeader = new String[gfs.size()]; |
|
65 |
int i = 0; |
|
66 |
for (GfsFile.PropertyDefn item : gfs) { |
|
67 |
theHeader[i++] = item.getName(); |
|
68 |
} |
|
69 |
this.header = theHeader; |
|
70 |
} |
|
71 |
return this.header; |
|
72 |
} |
|
73 |
|
|
74 |
@Override |
|
75 |
public int getColumnsCount() throws IOException { |
|
76 |
if (this.columns <= 0) { |
|
77 |
this.columns = this.getHeader().length; |
|
78 |
} |
|
79 |
return this.columns; |
|
80 |
} |
|
81 |
|
|
82 |
@Override |
|
83 |
public List<String> read() throws IOException { |
|
84 |
List<String> values = this.read(currentElement); |
|
85 |
if (values == null) { |
|
86 |
return null; |
|
87 |
} |
|
88 |
currentElement += 1; |
|
89 |
return values; |
|
90 |
} |
|
91 |
|
|
92 |
public List<String> read(int rowNumber) throws IOException { |
|
93 |
if (rowNumber < gml.size()) { |
|
94 |
List<String> values = gml.get(rowNumber); |
|
95 |
return values; |
|
96 |
} |
|
97 |
return null; |
|
98 |
} |
|
99 |
|
|
100 |
@Override |
|
101 |
public void close() throws IOException { |
|
102 |
this.gml.close(); |
|
103 |
this.gml = null; |
|
104 |
this.gfs = null; |
|
105 |
} |
|
106 |
|
|
107 |
@Override |
|
108 |
public List<String> skip(int lines) throws IOException { |
|
109 |
this.currentElement += lines; |
|
110 |
if (this.currentElement > this.gml.size()) { |
|
111 |
return null; |
|
112 |
} |
|
113 |
return read(this.currentElement); |
|
114 |
} |
|
115 |
|
|
116 |
@Override |
|
117 |
public int getLine() { |
|
118 |
if (this.gml == null) { |
|
119 |
return 0; |
|
120 |
} |
|
121 |
return this.currentElement; |
|
122 |
} |
|
123 |
|
|
124 |
@Override |
|
125 |
public List<String> nextRowValues() { |
|
126 |
try { |
|
127 |
return this.read(); |
|
128 |
} catch (IOException ex) { |
|
129 |
throw new RuntimeException(ex); |
|
130 |
} |
|
131 |
} |
|
132 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/xml/XmlCommons.java | ||
---|---|---|
1 |
package org.gvsig.fmap.dal.store.csv.xml; |
|
2 |
|
|
3 |
import java.io.BufferedReader; |
|
4 |
import java.io.File; |
|
5 |
import java.io.FileInputStream; |
|
6 |
import java.io.InputStream; |
|
7 |
import java.io.InputStreamReader; |
|
8 |
import java.nio.charset.Charset; |
|
9 |
import org.apache.commons.io.IOUtils; |
|
10 |
import org.apache.commons.io.input.BOMInputStream; |
|
11 |
import org.apache.commons.lang3.StringUtils; |
|
12 |
import org.apache.tika.detect.AutoDetectReader; |
|
13 |
import org.gvsig.tools.ToolsLocator; |
|
14 |
import org.gvsig.tools.i18n.I18nManager; |
|
15 |
import org.gvsig.tools.task.SimpleTaskStatus; |
|
16 |
import org.xml.sax.InputSource; |
|
17 |
|
|
18 |
/** |
|
19 |
* |
|
20 |
* @author jjdelcerro |
|
21 |
*/ |
|
22 |
@SuppressWarnings("UseSpecificCatch") |
|
23 |
public class XmlCommons { |
|
24 |
|
|
25 |
public static Charset detectCharset(InputStream is) { |
|
26 |
try { |
|
27 |
AutoDetectReader reader = new AutoDetectReader(is); |
|
28 |
return reader.getCharset(); |
|
29 |
} catch(Throwable t) { |
|
30 |
return null; |
|
31 |
} |
|
32 |
} |
|
33 |
|
|
34 |
public static String detectCharsetName(InputStream is) { |
|
35 |
Charset charset = detectCharset(is); |
|
36 |
if( charset==null ) { |
|
37 |
return null; |
|
38 |
} |
|
39 |
return charset.name(); |
|
40 |
} |
|
41 |
|
|
42 |
public static InputSource openReader(File xmlfile, Charset charset) { |
|
43 |
try { |
|
44 |
FileInputStream fis = new FileInputStream(xmlfile); |
|
45 |
|
|
46 |
InputSource is = new InputSource(); |
|
47 |
is.setPublicId(xmlfile.getAbsolutePath()); |
|
48 |
is.setByteStream(fis); |
|
49 |
if( charset!=null ) { |
|
50 |
is.setEncoding(charset.name()); |
|
51 |
} |
|
52 |
return openReader(is); |
|
53 |
} catch(Throwable t) { |
|
54 |
throw new RuntimeException("Can't open xml input stream.",t); |
|
55 |
} |
|
56 |
} |
|
57 |
|
|
58 |
public static InputSource openReader(InputStream xml, Charset charset) { |
|
59 |
InputSource is = new InputSource(); |
|
60 |
is.setByteStream(xml); |
|
61 |
if( charset!=null ) { |
|
62 |
is.setEncoding(charset.name()); |
|
63 |
} |
|
64 |
return openReader(is); |
|
65 |
} |
|
66 |
|
|
67 |
public static InputSource openReader(InputSource is) { |
|
68 |
try { |
|
69 |
if(StringUtils.isBlank(is.getEncoding())){ |
|
70 |
// EncodingDetector encodingDetector = TikaConfig.getDefaultConfig().getEncodingDetector(); |
|
71 |
// BufferedInputStream bis = new BufferedInputStream(is.getByteStream()); |
|
72 |
// Charset charset = encodingDetector.detect(bis, new Metadata()); |
|
73 |
// is.setEncoding(charset.name()); |
|
74 |
// is.setByteStream(bis); |
|
75 |
AutoDetectReader reader = new AutoDetectReader(is.getByteStream()); |
|
76 |
is.setCharacterStream(reader); |
|
77 |
is.setEncoding(reader.getCharset().name()); |
|
78 |
} else { |
|
79 |
BOMInputStream bomIs = new BOMInputStream(is.getByteStream()); |
|
80 |
is.setByteStream(bomIs); |
|
81 |
InputStreamReader reader = new InputStreamReader( |
|
82 |
is.getByteStream(), |
|
83 |
is.getEncoding() |
|
84 |
); |
|
85 |
is.setCharacterStream(reader); |
|
86 |
} |
|
87 |
return is; |
|
88 |
} catch(Throwable t) { |
|
89 |
throw new RuntimeException("Can't open xml input stream.",t); |
|
90 |
} |
|
91 |
} |
|
92 |
|
|
93 |
public static long countLines(File xml, Charset charset, SimpleTaskStatus status) { |
|
94 |
try { |
|
95 |
FileInputStream fis = new FileInputStream(xml); |
|
96 |
return countLines(fis, charset, status); |
|
97 |
} catch(Throwable t) { |
|
98 |
throw new RuntimeException("Can't count lines.",t); |
|
99 |
} |
|
100 |
} |
|
101 |
|
|
102 |
public static long countLines(InputStream xml, Charset charset, SimpleTaskStatus status) { |
|
103 |
try { |
|
104 |
long count = 0; |
|
105 |
// Reader reader = null; |
|
106 |
BufferedReader br = null; |
|
107 |
status.setIndeterminate(); |
|
108 |
status.setCurValue(0); |
|
109 |
try { |
|
110 |
InputSource is = new InputSource(xml); |
|
111 |
if( charset!=null ) { |
|
112 |
is.setEncoding(charset.name()); |
|
113 |
} |
|
114 |
is = openReader(is); |
|
115 |
br = new BufferedReader(is.getCharacterStream()); |
|
116 |
int n = 1; |
|
117 |
I18nManager i18n = ToolsLocator.getI18nManager(); |
|
118 |
while( br.readLine()!=null ) { |
|
119 |
if(count%n == 0){ |
|
120 |
status.message(i18n.getTranslation("_Calculating_lines")); |
|
121 |
status.setCurValue(count); |
|
122 |
} |
|
123 |
// status.incrementCurrentValue(); |
|
124 |
|
|
125 |
count++; |
|
126 |
if(count > 100000){ |
|
127 |
n = 10000; |
|
128 |
} else if(count > 10000){ |
|
129 |
n = 1000; |
|
130 |
} else if(count > 1000){ |
|
131 |
n = 100; |
|
132 |
} else if(count > 100){ |
|
133 |
n = 10; |
|
134 |
} |
|
135 |
} |
|
136 |
} finally { |
|
137 |
IOUtils.closeQuietly(br); |
|
138 |
} |
|
139 |
return count; |
|
140 |
} catch(Throwable t) { |
|
141 |
throw new RuntimeException("Can't count lines.",t); |
|
142 |
} |
|
143 |
} |
|
144 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/xml/xmlinfo/XMLInfoImpl.java | ||
---|---|---|
1 |
package org.gvsig.fmap.dal.store.csv.xml.xmlinfo; |
|
2 |
|
|
3 |
import java.nio.charset.Charset; |
|
4 |
import java.util.Collection; |
|
5 |
import java.util.LinkedHashMap; |
|
6 |
import java.util.Locale; |
|
7 |
import java.util.Map; |
|
8 |
|
|
9 |
/** |
|
10 |
* |
|
11 |
* @author jjdelcerro |
|
12 |
*/ |
|
13 |
public class XMLInfoImpl { |
|
14 |
|
|
15 |
private final Map<String,XMLAttributeInfoImpl> tags; |
|
16 |
private long countLines; |
|
17 |
private Charset charset; |
|
18 |
private Locale locale; |
|
19 |
|
|
20 |
public XMLInfoImpl() { |
|
21 |
this.tags = new LinkedHashMap<>(); |
|
22 |
this.countLines = -1; |
|
23 |
} |
|
24 |
|
|
25 |
public XMLAttributeInfoImpl getTag(String path) { |
|
26 |
return this.tags.get(path); |
|
27 |
} |
|
28 |
|
|
29 |
public Collection<String> getTagsPaths() { |
|
30 |
return this.tags.keySet(); |
|
31 |
} |
|
32 |
|
|
33 |
public void addTag(XMLAttributeInfoImpl info) { |
|
34 |
tags.put(info.getPath(), info); |
|
35 |
} |
|
36 |
|
|
37 |
public XMLAttributeInfoImpl getTagInfo(String path) { |
|
38 |
return this.tags.get(path); |
|
39 |
} |
|
40 |
|
|
41 |
public void setCountLines(long countlines) { |
|
42 |
this.countLines = countlines; |
|
43 |
} |
|
44 |
|
|
45 |
public long getCountLines() { |
|
46 |
return this.countLines; |
|
47 |
} |
|
48 |
|
|
49 |
public Charset getCharset() { |
|
50 |
return this.charset; |
|
51 |
} |
|
52 |
|
|
53 |
public void setCharset(Charset charset) { |
|
54 |
this.charset = charset; |
|
55 |
} |
|
56 |
|
|
57 |
public Locale getLocale() { |
|
58 |
return this.locale; |
|
59 |
} |
|
60 |
|
|
61 |
public void setLocale(Locale locale) { |
|
62 |
this.locale = locale; |
|
63 |
} |
|
64 |
|
|
65 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/xml/xmlinfo/XMLAttributeInfoImpl.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license header, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.csv.xml.xmlinfo; |
|
7 |
|
|
8 |
import java.util.HashMap; |
|
9 |
import java.util.Locale; |
|
10 |
import java.util.Map; |
|
11 |
import org.apache.commons.io.FilenameUtils; |
|
12 |
import org.apache.commons.lang3.mutable.MutableInt; |
|
13 |
import org.gvsig.fmap.dal.DALLocator; |
|
14 |
import org.gvsig.fmap.dal.DataTypes; |
|
15 |
import org.gvsig.fmap.dal.feature.DataTypeDetector; |
|
16 |
import org.gvsig.fmap.geom.type.GeometryType; |
|
17 |
import org.gvsig.tools.ToolsLocator; |
|
18 |
import org.gvsig.tools.dataTypes.DataType; |
|
19 |
import org.gvsig.tools.dataTypes.DataTypesManager; |
|
20 |
|
|
21 |
/** |
|
22 |
* |
|
23 |
* @author jjdelcerro |
|
24 |
*/ |
|
25 |
public class XMLAttributeInfoImpl { |
|
26 |
|
|
27 |
private String path; |
|
28 |
private String name; |
|
29 |
private int type; |
|
30 |
private Map<String, MutableInt> childCounts; |
|
31 |
private Map<String, Integer> childCountsMax; |
|
32 |
private Map<String, String> childIds; |
|
33 |
private DataTypeDetector dataTypeDetector; |
|
34 |
private boolean isAttr; |
|
35 |
private String ns; |
|
36 |
|
|
37 |
protected XMLAttributeInfoImpl(Locale locale) { |
|
38 |
this.path = null; |
|
39 |
this.name = null; |
|
40 |
this.type = DataTypes.STRING; |
|
41 |
this.childCounts = new HashMap<>(); |
|
42 |
this.childIds = new HashMap<>(); |
|
43 |
this.childCountsMax = new HashMap<>(); |
|
44 |
this.dataTypeDetector = DALLocator.getDataManager().createDataTypeDetector(locale); |
|
45 |
this.isAttr = false; |
|
46 |
} |
|
47 |
|
|
48 |
public XMLAttributeInfoImpl(Locale locale, String path, String ns) { |
|
49 |
this(locale); |
|
50 |
this.path = path; |
|
51 |
this.name = FilenameUtils.getBaseName(path); |
|
52 |
this.ns = ns; |
|
53 |
// if( StringUtils.contains(this.path, "/ALCOHOL/") ) { |
|
54 |
// System.out.println("### TAG: "+this.path); |
|
55 |
// } |
|
56 |
} // if( StringUtils.contains(this.path, "/ALCOHOL/") ) { |
|
57 |
// System.out.println("### TAG: "+this.path); |
|
58 |
// } |
|
59 |
|
|
60 |
public XMLAttributeInfoImpl(Locale locale, String name, int type) { |
|
61 |
this(locale); |
|
62 |
this.name = name; |
|
63 |
this.setType(type); |
|
64 |
} |
|
65 |
|
|
66 |
public String getName() { |
|
67 |
return name; |
|
68 |
} |
|
69 |
|
|
70 |
public String getPath() { |
|
71 |
return path; |
|
72 |
} |
|
73 |
|
|
74 |
public void setType(int type) { |
|
75 |
this.type = type; |
|
76 |
this.dataTypeDetector = null; |
|
77 |
} |
|
78 |
|
|
79 |
public int getType() { |
|
80 |
if (this.dataTypeDetector == null) { |
|
81 |
return this.type; |
|
82 |
} |
|
83 |
int x = this.dataTypeDetector.getDataType().getType(); |
|
84 |
if (x == DataTypes.UNKNOWN) { |
|
85 |
return DataTypes.STRING; |
|
86 |
} |
|
87 |
return x; |
|
88 |
} |
|
89 |
|
|
90 |
public int getPrecision() { |
|
91 |
if (this.dataTypeDetector == null) { |
|
92 |
return DataType.PRECISION_NONE; |
|
93 |
} |
|
94 |
return this.dataTypeDetector.getDataType().getPrecision(); |
|
95 |
} |
|
96 |
|
|
97 |
public int getScale() { |
|
98 |
if (this.dataTypeDetector == null) { |
|
99 |
return DataType.SCALE_NONE; |
|
100 |
} |
|
101 |
return this.dataTypeDetector.getDataType().getScale(); |
|
102 |
} |
|
103 |
|
|
104 |
public int getSize() { |
|
105 |
if (this.dataTypeDetector == null) { |
|
106 |
return 0; |
|
107 |
} |
|
108 |
if (this.getType() != DataTypes.STRING) { |
|
109 |
return 0; |
|
110 |
} |
|
111 |
int size = this.dataTypeDetector.getDataType().getDisplaySize(); |
|
112 |
if (size > 0) { |
|
113 |
size = ((size + 10) / 10) * 10; |
|
114 |
} |
|
115 |
return size; |
|
116 |
} |
|
117 |
|
|
118 |
public GeometryType getGeometryType() { |
|
119 |
if (this.dataTypeDetector == null) { |
|
120 |
return null; |
|
121 |
} |
|
122 |
return this.dataTypeDetector.getDataType().getGeometryType(); |
|
123 |
} |
|
124 |
|
|
125 |
public String getNs() { |
|
126 |
return ns; |
|
127 |
} |
|
128 |
|
|
129 |
|
|
130 |
|
|
131 |
@Override |
|
132 |
public String toString() { |
|
133 |
DataTypesManager dataTypeManager = ToolsLocator.getDataTypesManager(); |
|
134 |
StringBuilder builder = new StringBuilder(); |
|
135 |
builder.append(name); |
|
136 |
builder.append(" "); |
|
137 |
builder.append(dataTypeManager.getTypeName(this.getType())); |
|
138 |
switch (this.getType()) { |
|
139 |
case DataTypes.STRING: |
|
140 |
builder.append("("); |
|
141 |
builder.append(this.getSize()); |
|
142 |
builder.append(")"); |
|
143 |
break; |
|
144 |
case DataTypes.DECIMAL: |
|
145 |
builder.append("("); |
|
146 |
builder.append(this.getPrecision()); |
|
147 |
builder.append(","); |
|
148 |
builder.append(this.getScale()); |
|
149 |
builder.append(")"); |
|
150 |
break; |
|
151 |
case DataTypes.GEOMETRY: |
|
152 |
builder.append("("); |
|
153 |
builder.append(this.getGeometryType().getFullName()); |
|
154 |
builder.append(")"); |
|
155 |
break; |
|
156 |
} |
|
157 |
return builder.toString(); |
|
158 |
} |
|
159 |
|
|
160 |
public void incrChildCount(String childName) { |
|
161 |
MutableInt count = this.childCounts.get(childName); |
|
162 |
if (count == null) { |
|
163 |
count = new MutableInt(0); |
|
164 |
this.childCounts.put(childName, count); |
|
165 |
} |
|
166 |
count.increment(); |
|
167 |
} |
|
168 |
|
|
169 |
public void setLastChildID(String childName, String idvalue) { |
|
170 |
this.childIds.put(childName, idvalue); |
|
171 |
} |
|
172 |
|
|
173 |
public String getLastChildID(String childName) { |
|
174 |
return this.childIds.get(childName); |
|
175 |
} |
|
176 |
|
|
177 |
public void consolidateChildCounters() { |
|
178 |
for (Map.Entry<String, MutableInt> entry : childCounts.entrySet()) { |
|
179 |
String childName = entry.getKey(); |
|
180 |
MutableInt count = entry.getValue(); |
|
181 |
int countmax = this.childCountsMax.getOrDefault(childName, -1); |
|
182 |
if (countmax < count.getValue() || countmax < 0) { |
|
183 |
this.childCountsMax.put(childName, count.getValue()); |
|
184 |
} |
|
185 |
childCounts.get(childName).setValue(0); |
|
186 |
} |
|
187 |
} |
|
188 |
|
|
189 |
public int getMaxChildCount(String childName) { |
|
190 |
int countmax = this.childCountsMax.getOrDefault(childName, 0); |
|
191 |
return countmax; |
|
192 |
} |
|
193 |
|
|
194 |
public void setSize(int size) { |
|
195 |
if (this.dataTypeDetector == null) { |
|
196 |
return; |
|
197 |
} |
|
198 |
DataTypeDetector.DataTypeDetected detectedType = this.dataTypeDetector.getDataType(); |
|
199 |
detectedType.setDisplaySize(size); |
|
200 |
} |
|
201 |
|
|
202 |
public void addValue(String value) { |
|
203 |
if (this.dataTypeDetector == null) { |
|
204 |
return; |
|
205 |
} |
|
206 |
this.dataTypeDetector.addValue(value); |
|
207 |
} |
|
208 |
|
|
209 |
public boolean hasChilds() { |
|
210 |
return !this.childCounts.isEmpty(); |
|
211 |
} |
|
212 |
|
|
213 |
public int getChildsCount(String name) { |
|
214 |
MutableInt count = this.childCounts.get(name); |
|
215 |
if(count == null){ |
|
216 |
return 0; |
|
217 |
} |
|
218 |
return count.getValue(); |
|
219 |
} |
|
220 |
|
|
221 |
public void setIsAttr(boolean isAttr) { |
|
222 |
this.isAttr = isAttr; |
|
223 |
} |
|
224 |
|
|
225 |
public boolean isAttr() { |
|
226 |
return this.isAttr; |
|
227 |
} |
|
228 |
|
|
229 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/xml/StructureExtractorImpl.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license header, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.csv.xml; |
|
7 |
|
|
8 |
import java.io.File; |
|
9 |
import java.io.FileNotFoundException; |
|
10 |
import java.io.IOException; |
|
11 |
import java.io.InputStream; |
|
12 |
import java.io.Reader; |
|
13 |
import java.nio.charset.Charset; |
|
14 |
import java.util.ArrayList; |
|
15 |
import java.util.List; |
|
16 |
import java.util.Locale; |
|
17 |
import javax.xml.parsers.SAXParser; |
|
18 |
import javax.xml.parsers.SAXParserFactory; |
|
19 |
import org.apache.commons.lang3.StringUtils; |
|
20 |
import org.apache.tika.utils.CharsetUtils; |
|
21 |
import org.gvsig.fmap.dal.DALLocator; |
|
22 |
import org.gvsig.fmap.dal.DataManager; |
|
23 |
import org.gvsig.fmap.dal.store.csv.xml.xmlinfo.XMLAttributeInfoImpl; |
|
24 |
import org.gvsig.fmap.dal.store.csv.xml.xmlinfo.XMLInfoImpl; |
|
25 |
import org.gvsig.tools.ToolsLocator; |
|
26 |
import org.gvsig.tools.i18n.I18nManager; |
|
27 |
import org.gvsig.tools.task.SimpleTaskStatus; |
|
28 |
import org.slf4j.Logger; |
|
29 |
import org.slf4j.LoggerFactory; |
|
30 |
import org.xml.sax.Attributes; |
|
31 |
import org.xml.sax.InputSource; |
|
32 |
import org.xml.sax.Locator; |
|
33 |
import org.xml.sax.SAXException; |
|
34 |
import org.xml.sax.helpers.DefaultHandler; |
|
35 |
|
|
36 |
/** |
|
37 |
* |
|
38 |
* @author jjdelcerro |
|
39 |
*/ |
|
40 |
public class StructureExtractorImpl { //implements StructureExtractor { |
|
41 |
|
|
42 |
private static final Logger LOGGER = LoggerFactory.getLogger(StructureExtractorImpl.class); |
|
43 |
|
|
44 |
@SuppressWarnings("UseSpecificCatch") |
|
45 |
private void extractTags(XMLInfoImpl xmlinfo, Reader reader, SimpleTaskStatus status) { |
|
46 |
if( reader == null ) { |
|
47 |
throw new IllegalArgumentException("reader is null"); |
|
48 |
} |
|
49 |
try { |
|
50 |
final DataManager dataManager = DALLocator.getDataManager(); |
|
51 |
I18nManager i18n = ToolsLocator.getI18nManager(); |
|
52 |
status.message(i18n.getTranslation("_Reading_xml")+" 1/5"); |
|
53 |
status.setRangeOfValues(0, xmlinfo.getCountLines()); |
|
54 |
|
|
55 |
SAXParserFactory spf = SAXParserFactory.newInstance(); |
|
56 |
spf.setNamespaceAware(true); |
|
57 |
SAXParser saxParser = spf.newSAXParser(); |
|
58 |
InputSource is = new InputSource(reader); |
|
59 |
|
|
60 |
List<String> path = new ArrayList<>(); |
|
61 |
|
|
62 |
saxParser.parse(is, new DefaultHandler() { |
|
63 |
private Locator locator; |
|
64 |
int size; |
|
65 |
int refreshInterval = 1; |
|
66 |
StringBuilder chars = new StringBuilder(); |
|
67 |
|
|
68 |
@Override |
|
69 |
public void setDocumentLocator(Locator locator) { |
|
70 |
this.locator = locator; |
|
71 |
} |
|
72 |
|
|
73 |
@Override |
|
74 |
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { |
|
75 |
int line = this.locator.getLineNumber(); |
|
76 |
int column = this.locator.getColumnNumber()-2-localName.length(); |
|
77 |
|
|
78 |
if(line % refreshInterval == 0) { |
|
79 |
status.setCurValue(line); |
|
80 |
} |
|
81 |
|
|
82 |
if(line > 100000){ |
|
83 |
refreshInterval = 10000; |
|
84 |
} else if(line > 10000){ |
|
85 |
refreshInterval = 1000; |
|
86 |
} else if(line > 1000){ |
|
87 |
refreshInterval = 100; |
|
88 |
} else if(line > 100){ |
|
89 |
refreshInterval = 10; |
|
90 |
} |
|
91 |
|
|
92 |
|
|
93 |
String idvalue = dataManager.createUniqueID(); |
|
94 |
|
|
95 |
path.add(localName); |
|
96 |
String path_s = StringUtils.join(path, "/"); |
|
97 |
XMLAttributeInfoImpl info = xmlinfo.getTag(path_s); |
|
98 |
if( info == null ) { |
|
99 |
info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s, getNs(qName)); |
|
100 |
xmlinfo.addTag(info); |
|
101 |
|
|
102 |
} |
|
103 |
if( path.size()>1 ) { |
|
104 |
List<String> parentpath = path.subList(0, path.size()-1); |
|
105 |
String parentpath_s = StringUtils.join(parentpath, "/"); |
|
106 |
XMLAttributeInfoImpl parentinfo = xmlinfo.getTag(parentpath_s); |
|
107 |
parentinfo.incrChildCount(localName); |
|
108 |
parentinfo.setLastChildID(localName, idvalue); |
|
109 |
} |
|
110 |
|
|
111 |
for (int i = 0; i < attributes.getLength(); i++) { |
|
112 |
String name = attributes.getLocalName(i); |
|
113 |
String value = attributes.getValue(i); |
|
114 |
String idvalueChild = dataManager.createUniqueID(); |
|
115 |
XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name); |
|
116 |
if( infoChild == null ) { |
|
117 |
infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name, getNs(qName)); |
|
118 |
infoChild.setIsAttr(true); |
|
119 |
xmlinfo.addTag(infoChild); |
|
120 |
} |
|
121 |
info.incrChildCount(infoChild.getName()); |
|
122 |
info.setLastChildID(infoChild.getName(), idvalueChild); |
|
123 |
infoChild.addValue(value); |
|
124 |
} |
|
125 |
chars.setLength(0); |
|
126 |
} |
|
127 |
|
|
128 |
@Override |
|
129 |
public void endElement(String uri, String localName, String qName) throws SAXException { |
|
130 |
int line = this.locator.getLineNumber(); |
|
131 |
|
|
132 |
// status.setCurValue(line); |
|
133 |
|
|
134 |
String path_s = StringUtils.join(path, "/"); |
|
135 |
XMLAttributeInfoImpl info = xmlinfo.getTag(path_s); |
|
136 |
if( info == null ) { |
|
137 |
info = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s, getNs(qName)); |
|
138 |
xmlinfo.addTag(info); |
|
139 |
} |
|
140 |
|
|
141 |
XMLAttributeInfoImpl parentinfo = null; |
|
142 |
if( path.size()>1 ) { |
|
143 |
List<String> parentpath = path.subList(0, path.size()-1); |
|
144 |
String parentpath_s = StringUtils.join(parentpath, "/"); |
|
145 |
parentinfo = xmlinfo.getTag(parentpath_s); |
|
146 |
} |
|
147 |
|
|
148 |
if( info.hasChilds() || (parentinfo != null && (parentinfo.getChildsCount(localName)>1))) { |
|
149 |
String value = this.chars.toString(); |
|
150 |
if( StringUtils.isNotBlank(value) ) { |
|
151 |
String name = info.getName()+"$v"; |
|
152 |
String idvalueChild = dataManager.createUniqueID(); |
|
153 |
XMLAttributeInfoImpl infoChild = xmlinfo.getTag(path_s+"/"+name); |
|
154 |
if( infoChild == null ) { |
|
155 |
infoChild = new XMLAttributeInfoImpl(xmlinfo.getLocale(), path_s+"/"+name, getNs(qName)); |
|
156 |
xmlinfo.addTag(infoChild); |
|
157 |
} |
|
158 |
info.incrChildCount(infoChild.getName()); |
|
159 |
info.setLastChildID(infoChild.getName(), idvalueChild); |
|
160 |
infoChild.addValue(value); |
|
161 |
} |
|
162 |
} else { |
|
163 |
String value = this.chars.toString(); |
|
164 |
info.addValue(value); |
|
165 |
} |
|
166 |
info.consolidateChildCounters(); |
|
167 |
path.remove(path.size()-1); |
|
168 |
chars.setLength(0); |
|
169 |
} |
|
170 |
|
|
171 |
@Override |
|
172 |
public void characters(char[] ch, int start, int length) throws SAXException { |
|
173 |
int line = this.locator.getLineNumber(); |
|
174 |
|
|
175 |
// status.setCurValue(line); |
|
176 |
this.chars.append(ch, start, length); |
|
177 |
} |
|
178 |
|
|
179 |
|
|
180 |
}); |
|
181 |
} catch (Exception ex) { |
|
182 |
throw new RuntimeException("Can't extract tags.", ex); |
|
183 |
} |
|
184 |
} |
|
185 |
|
|
186 |
public XMLInfoImpl extractStructure(File xml, Charset charset, Locale locale, SimpleTaskStatus status) throws FileNotFoundException, IOException { |
|
187 |
XMLInfoImpl xmlinfo = new XMLInfoImpl(); |
|
188 |
xmlinfo.setLocale(locale); |
|
189 |
long count = XmlCommons.countLines(xml, charset, status); |
|
190 |
xmlinfo.setCountLines(count); |
|
191 |
InputSource is = XmlCommons.openReader(xml,charset); |
|
192 |
return extractStructure(is, xmlinfo, status); |
|
193 |
} |
|
194 |
|
|
195 |
public XMLInfoImpl extractStructure(InputStream xml, Charset charset, Locale locale, SimpleTaskStatus status) throws IOException { |
|
196 |
XMLInfoImpl xmlinfo = new XMLInfoImpl(); |
|
197 |
xmlinfo.setLocale(locale); |
|
198 |
xmlinfo.setCountLines(-1); |
|
199 |
InputSource is = XmlCommons.openReader(xml, charset); |
|
200 |
return extractStructure(is, xmlinfo, status); |
|
201 |
} |
|
202 |
|
|
203 |
public XMLInfoImpl extractStructure(Reader reader, Locale locale, SimpleTaskStatus status) { |
|
204 |
XMLInfoImpl xmlinfo = new XMLInfoImpl(); |
|
205 |
xmlinfo.setLocale(locale); |
|
206 |
InputSource is = new InputSource(reader); |
|
207 |
return extractStructure(is, xmlinfo, status); |
|
208 |
} |
|
209 |
|
|
210 |
public XMLInfoImpl extractStructure(InputSource is, XMLInfoImpl xmlinfo, SimpleTaskStatus status) { |
|
211 |
|
|
212 |
if( xmlinfo.getCharset()==null ) { |
|
213 |
xmlinfo.setCharset(CharsetUtils.forName(is.getEncoding())); |
|
214 |
} |
|
215 |
|
|
216 |
extractTags(xmlinfo, is.getCharacterStream(), status); |
|
217 |
|
|
218 |
return xmlinfo; |
|
219 |
} |
|
220 |
|
|
221 |
private String getNs(String s){ |
|
222 |
String[] ss = s.split(":"); |
|
223 |
if(ss.length == 1){ |
|
224 |
return null; |
|
225 |
} |
|
226 |
return ss[0]; |
|
227 |
} |
|
228 |
|
|
229 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/xml/GfsFile.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license header, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.csv.xml; |
|
7 |
|
|
8 |
import java.io.BufferedWriter; |
|
9 |
import java.io.File; |
|
10 |
import java.io.FileWriter; |
|
11 |
import java.io.IOException; |
|
12 |
import java.util.ArrayList; |
|
13 |
import java.util.Arrays; |
|
14 |
import java.util.Iterator; |
|
15 |
import java.util.List; |
|
16 |
import java.util.Locale; |
|
17 |
import java.util.Map; |
|
18 |
import javax.xml.parsers.SAXParser; |
|
19 |
import javax.xml.parsers.SAXParserFactory; |
|
20 |
import org.apache.commons.collections4.map.LinkedMap; |
|
21 |
import org.apache.commons.io.FilenameUtils; |
|
22 |
import org.apache.commons.io.IOUtils; |
|
23 |
import org.apache.commons.lang3.StringUtils; |
|
24 |
import org.gvsig.fmap.dal.store.csv.xml.GfsFile.PropertyDefn; |
|
25 |
import org.gvsig.fmap.dal.store.csv.xml.xmlinfo.XMLAttributeInfoImpl; |
|
26 |
import org.gvsig.fmap.dal.store.csv.xml.xmlinfo.XMLInfoImpl; |
|
27 |
import org.gvsig.tools.ToolsLocator; |
|
28 |
import org.gvsig.tools.dataTypes.DataTypes; |
|
29 |
import org.gvsig.tools.dataTypes.DataTypesManager; |
|
30 |
import org.gvsig.tools.library.impl.DefaultLibrariesInitializer; |
|
31 |
import org.gvsig.tools.task.SimpleTaskStatus; |
|
32 |
import org.gvsig.tools.util.IsEmpty; |
|
33 |
import org.gvsig.tools.util.Size; |
|
34 |
import org.slf4j.Logger; |
|
35 |
import org.slf4j.LoggerFactory; |
|
36 |
import org.xml.sax.Attributes; |
|
37 |
import org.xml.sax.InputSource; |
|
38 |
import org.xml.sax.Locator; |
|
39 |
import org.xml.sax.SAXException; |
|
40 |
import org.xml.sax.helpers.DefaultHandler; |
|
41 |
|
|
42 |
/** |
|
43 |
* |
|
44 |
* @author fdiaz |
|
45 |
*/ |
|
46 |
public class GfsFile implements Iterable<PropertyDefn>, IsEmpty, Size { |
|
47 |
|
|
48 |
private static final Logger LOGGER = LoggerFactory.getLogger(GfsFile.class); |
|
49 |
|
|
50 |
private static final String PROPERTY_NAME_SEPARATOR = "_"; |
|
51 |
private static final String PROPERTY_PATH_SEPARATOR = "|"; |
|
52 |
private static final String PROPERTY_ATTRIBUTE_SEPARATOR = "@"; |
|
53 |
|
|
54 |
public static class PropertyDefn { |
|
55 |
private String name; |
|
56 |
private String elementPath; |
|
57 |
private int type; |
|
58 |
private int width; |
|
59 |
private boolean isAttr; |
|
60 |
|
|
61 |
public PropertyDefn() { |
|
62 |
} |
|
63 |
|
|
64 |
public PropertyDefn(String name, String elementPath, int type, int width, boolean isAttr) { |
|
65 |
this.name = name; |
|
66 |
this.elementPath = elementPath; |
|
67 |
this.type = type; |
|
68 |
this.width = width; |
|
69 |
this.isAttr = isAttr; |
|
70 |
} |
|
71 |
|
|
72 |
public String getName() { |
|
73 |
return name; |
|
74 |
} |
|
75 |
|
|
76 |
public void setName(String name) { |
|
77 |
this.name = name; |
|
78 |
} |
|
79 |
|
|
80 |
public String getElementPath() { |
|
81 |
return elementPath; |
|
82 |
} |
|
83 |
|
|
84 |
public void setElementPath(String elementPath) { |
|
85 |
this.elementPath = elementPath; |
|
86 |
} |
|
87 |
|
|
88 |
public int getType() { |
|
89 |
return type; |
|
90 |
} |
|
91 |
|
|
92 |
public void setType(int type) { |
|
93 |
this.type = type; |
|
94 |
} |
|
95 |
|
|
96 |
public int getWidth() { |
|
97 |
return width; |
|
98 |
} |
|
99 |
|
|
100 |
public void setWidth(int width) { |
|
101 |
this.width = width; |
|
102 |
} |
|
103 |
|
|
104 |
public boolean isAttr() { |
|
105 |
return isAttr; |
|
106 |
} |
|
107 |
|
|
108 |
public void setIsAttr(boolean b) { |
|
109 |
this.isAttr = b; |
|
110 |
} |
|
111 |
|
|
112 |
@Override |
|
113 |
public boolean equals(Object obj) { |
|
114 |
if(!(obj instanceof PropertyDefn)) { |
|
115 |
return false; |
|
116 |
} |
|
117 |
PropertyDefn other = (PropertyDefn)obj; |
|
118 |
if(!StringUtils.equals(this.getName(), other.getName())){ |
|
119 |
return false; |
|
120 |
} |
|
121 |
if(!StringUtils.equals(this.getElementPath(), other.getElementPath())){ |
|
122 |
return false; |
|
123 |
} |
|
124 |
if(this.getType()!=other.getType()){ |
|
125 |
return false; |
|
126 |
} |
|
127 |
if(this.getWidth()!=other.getWidth()){ |
|
128 |
return false; |
|
129 |
} |
|
130 |
if(this.isAttr()!=other.isAttr()){ |
|
131 |
return false; |
|
132 |
} |
|
133 |
return true; |
|
134 |
} |
|
135 |
} |
|
136 |
|
|
137 |
private String name; |
|
138 |
private List<String> geometryElementPaths; |
|
139 |
private String baseElementPath; |
|
140 |
private Map<String, PropertyDefn> properties; |
|
141 |
private int gmlVersion; |
|
142 |
|
|
143 |
public String getName() { |
|
144 |
return name; |
|
145 |
} |
|
146 |
|
|
147 |
public void setName(String name) { |
|
148 |
this.name = name; |
|
149 |
} |
|
150 |
|
|
151 |
public List<String> getGeometryElementPaths() { |
|
152 |
return geometryElementPaths; |
|
153 |
} |
|
154 |
|
|
155 |
public void setGeometryElementPaths(List<String> geometryElementPaths) { |
|
156 |
this.geometryElementPaths = geometryElementPaths; |
|
157 |
} |
|
158 |
|
|
159 |
public void addGeometryElementPath(String geometryElementPath) { |
|
160 |
if(StringUtils.isBlank(geometryElementPath)){ |
|
161 |
return; |
|
162 |
} |
|
163 |
if(this.geometryElementPaths == null){ |
|
164 |
this.geometryElementPaths = new ArrayList<>(); |
|
165 |
} |
|
166 |
this.geometryElementPaths.add(geometryElementPath); |
|
167 |
} |
|
168 |
|
|
169 |
public String getBaseElementPath() { |
|
170 |
return baseElementPath; |
|
171 |
} |
|
172 |
|
|
173 |
public void setBaseElementPath(String baseElementPath) { |
|
174 |
this.baseElementPath = baseElementPath; |
|
175 |
} |
|
176 |
|
|
177 |
private String findBaseElementPath(XMLInfoImpl xmlinfo) { |
|
178 |
String possibleMemberPath = null; |
|
179 |
for (String path : xmlinfo.getTagsPaths()) { |
|
180 |
XMLAttributeInfoImpl tag = xmlinfo.getTag(path); |
|
181 |
|
|
182 |
if (StringUtils.isBlank(possibleMemberPath) |
|
183 |
&& !tag.isAttr() |
|
184 |
&& !StringUtils.equalsIgnoreCase(tag.getNs(), "gml") |
|
185 |
&& !StringUtils.equalsIgnoreCase(path, "FeatureCollection/boundedBy") |
|
186 |
) { |
|
187 |
|
|
188 |
String[] ss = path.split("/"); |
|
189 |
if(ss.length == 2){ |
|
190 |
possibleMemberPath = path; |
|
191 |
} |
|
192 |
} |
|
193 |
if (path.equalsIgnoreCase("FeatureCollection/member")) { |
|
194 |
return path; |
|
195 |
} else if (path.equalsIgnoreCase("FeatureCollection/featureMember")) { |
|
196 |
return path; |
|
197 |
} |
|
198 |
} |
|
199 |
return possibleMemberPath; |
|
200 |
} |
|
201 |
|
|
202 |
private List<String> findGeometryElementPaths(XMLInfoImpl xmlinfo) { |
|
203 |
List<String> result = new ArrayList<>(); |
|
204 |
List<String> paths = new ArrayList<>(); |
|
205 |
for (String path : xmlinfo.getTagsPaths()) { |
|
206 |
XMLAttributeInfoImpl tag = xmlinfo.getTag(path); |
|
207 |
if(tag.isAttr()){ |
|
208 |
continue; |
|
209 |
} |
|
210 |
paths.add(path); |
|
211 |
} |
|
212 |
for (int i = 0; i < paths.size(); i++) { |
|
213 |
String path = paths.get(i); |
|
214 |
XMLAttributeInfoImpl tag = xmlinfo.getTag(path); |
|
215 |
if(i == paths.size()-1){ |
|
216 |
continue; |
|
217 |
} |
|
218 |
XMLAttributeInfoImpl nextTag = xmlinfo.getTag(paths.get(i+1)); |
|
219 |
|
|
220 |
String p = null; |
|
221 |
|
|
222 |
switch(nextTag.getName().toUpperCase()){ |
|
223 |
case "POINT": |
|
224 |
case "MULTIPOINT": |
|
225 |
case "LINESTRING": |
|
226 |
p=path; |
|
227 |
this.gmlVersion = 0; |
|
228 |
break; |
|
229 |
|
|
230 |
case "POLYGON": |
|
231 |
case "MULTIPOLYGON": |
|
232 |
p=path; |
|
233 |
this.gmlVersion = 2; |
|
234 |
break; |
|
235 |
case "SURFACE": |
|
236 |
case "MULTISURFACE": |
|
237 |
case "CURVE": |
|
238 |
case "MULTICURVE": |
|
239 |
p=path; |
|
240 |
this.gmlVersion = 3; |
|
241 |
} |
|
242 |
if(p!=null){ |
|
243 |
for (String s : result) { |
|
244 |
if(StringUtils.startsWithIgnoreCase(p, s)){ |
|
245 |
p=null; |
|
246 |
break; |
|
247 |
} |
|
248 |
} |
|
249 |
if (p != null) { |
|
250 |
result.add(p); |
|
251 |
} |
|
252 |
} |
|
253 |
} |
|
254 |
|
|
255 |
if(result.isEmpty()){ |
|
256 |
return null; |
|
257 |
} |
|
258 |
return result; |
|
259 |
} |
|
260 |
|
|
261 |
|
Also available in: Unified diff