svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / simplereader / SimpleReaderStoreProvider.java @ 47634
History | View | Annotate | Download (42.6 KB)
1 | 40846 | jjdelcerro | /**
|
---|---|---|---|
2 | * gvSIG. Desktop Geographic Information System.
|
||
3 | *
|
||
4 | * Copyright (C) 2007-2013 gvSIG Association.
|
||
5 | *
|
||
6 | 42775 | jjdelcerro | * This program is free software; you can redistribute it and/or modify it under
|
7 | * the terms of the GNU General Public License as published by the Free Software
|
||
8 | * Foundation; either version 3 of the License, or (at your option) any later
|
||
9 | * version.
|
||
10 | 40846 | jjdelcerro | *
|
11 | 42775 | jjdelcerro | * This program is distributed in the hope that it will be useful, but WITHOUT
|
12 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||
13 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||
14 | * details.
|
||
15 | 40846 | jjdelcerro | *
|
16 | 42775 | jjdelcerro | * You should have received a copy of the GNU General Public License along with
|
17 | * this program; if not, write to the Free Software Foundation, Inc., 51
|
||
18 | * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||
19 | 40846 | jjdelcerro | *
|
20 | 42775 | jjdelcerro | * For any additional information, do not hesitate to contact us at info AT
|
21 | * gvsig.com, or visit our website www.gvsig.com.
|
||
22 | 40846 | jjdelcerro | */
|
23 | 47634 | fdiaz | package org.gvsig.fmap.dal.store.simplereader; |
24 | 40846 | jjdelcerro | |
25 | 47634 | fdiaz | import org.gvsig.fmap.dal.store.csv.*; |
26 | 45836 | fdiaz | import java.io.Closeable; |
27 | 40846 | jjdelcerro | import java.io.File; |
28 | 45836 | fdiaz | import java.io.IOException; |
29 | 43283 | jjdelcerro | import java.io.InputStreamReader; |
30 | 47634 | fdiaz | import java.io.Reader; |
31 | 43425 | jjdelcerro | import java.net.URI; |
32 | 41062 | jjdelcerro | import java.net.URL; |
33 | 47199 | jjdelcerro | import java.nio.charset.StandardCharsets; |
34 | 40846 | jjdelcerro | import java.util.ArrayList; |
35 | import java.util.HashMap; |
||
36 | 41006 | jjdelcerro | import java.util.Iterator; |
37 | 40846 | jjdelcerro | import java.util.List; |
38 | 47199 | jjdelcerro | import java.util.Objects; |
39 | 45836 | fdiaz | import org.apache.commons.io.FileUtils; |
40 | 40846 | jjdelcerro | |
41 | 41006 | jjdelcerro | import org.apache.commons.io.FilenameUtils; |
42 | 45836 | fdiaz | import org.apache.commons.io.IOUtils; |
43 | 41876 | jjdelcerro | import org.apache.commons.lang3.StringUtils; |
44 | 40846 | jjdelcerro | import org.cresques.cts.IProjection; |
45 | import org.gvsig.fmap.dal.DALLocator; |
||
46 | import org.gvsig.fmap.dal.DataManager; |
||
47 | import org.gvsig.fmap.dal.DataServerExplorer; |
||
48 | import org.gvsig.fmap.dal.DataStore; |
||
49 | import org.gvsig.fmap.dal.DataStoreNotification; |
||
50 | import org.gvsig.fmap.dal.DataTypes; |
||
51 | import org.gvsig.fmap.dal.FileHelper; |
||
52 | 46608 | fdiaz | import org.gvsig.fmap.dal.exception.CloseException; |
53 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.exception.DataException; |
54 | import org.gvsig.fmap.dal.exception.InitializeException; |
||
55 | import org.gvsig.fmap.dal.exception.OpenException; |
||
56 | import org.gvsig.fmap.dal.exception.ReadException; |
||
57 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
58 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.EditableFeatureType; |
59 | 41059 | jjdelcerro | import org.gvsig.fmap.dal.feature.Feature; |
60 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor; |
61 | 45727 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureQuery; |
62 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureSet; |
63 | 41059 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureStore; |
64 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.FeatureType; |
65 | 45836 | fdiaz | import org.gvsig.fmap.dal.feature.exception.CreateFeatureException; |
66 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.feature.exception.PerformEditingException; |
67 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.spi.FeatureProvider; |
68 | 45836 | fdiaz | import org.gvsig.fmap.dal.feature.spi.FeatureReferenceProviderServices; |
69 | 45727 | jjdelcerro | import org.gvsig.fmap.dal.feature.spi.FeatureSetProvider; |
70 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.feature.spi.memory.AbstractMemoryStoreProvider; |
71 | 41006 | jjdelcerro | import org.gvsig.fmap.dal.resource.ResourceAction; |
72 | 40846 | jjdelcerro | import org.gvsig.fmap.dal.resource.file.FileResource; |
73 | import org.gvsig.fmap.dal.resource.spi.ResourceConsumer; |
||
74 | import org.gvsig.fmap.dal.resource.spi.ResourceProvider; |
||
75 | import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorer; |
||
76 | import org.gvsig.fmap.dal.serverexplorer.filesystem.FilesystemServerExplorerParameters; |
||
77 | import org.gvsig.fmap.dal.spi.DataStoreProviderServices; |
||
78 | 45727 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.simplereaders.AbstractSimpleReader; |
79 | 45685 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReaderSuperCSV; |
80 | 41617 | jjdelcerro | import org.gvsig.fmap.dal.store.csv.simplereaders.SimpleReader; |
81 | 40846 | jjdelcerro | import org.gvsig.fmap.geom.Geometry; |
82 | import org.gvsig.fmap.geom.GeometryLocator; |
||
83 | import org.gvsig.fmap.geom.GeometryManager; |
||
84 | 47199 | jjdelcerro | import org.gvsig.fmap.geom.SpatialIndex; |
85 | import org.gvsig.fmap.geom.SpatialIndexFactory; |
||
86 | 40846 | jjdelcerro | import org.gvsig.fmap.geom.primitive.Envelope; |
87 | 41006 | jjdelcerro | import org.gvsig.fmap.geom.primitive.Point; |
88 | 40846 | jjdelcerro | import org.gvsig.tools.ToolsLocator; |
89 | 44669 | jjdelcerro | import org.gvsig.tools.dataTypes.Coercion; |
90 | 41006 | jjdelcerro | import org.gvsig.tools.dispose.DisposableIterator; |
91 | 40846 | jjdelcerro | import org.gvsig.tools.dynobject.exception.DynFieldNotFoundException; |
92 | 41006 | jjdelcerro | import org.gvsig.tools.evaluator.AbstractEvaluator; |
93 | import org.gvsig.tools.evaluator.EvaluatorData; |
||
94 | import org.gvsig.tools.evaluator.EvaluatorException; |
||
95 | 41059 | jjdelcerro | import org.gvsig.tools.exception.BaseException; |
96 | 40846 | jjdelcerro | import org.gvsig.tools.exception.NotYetImplemented; |
97 | import org.gvsig.tools.persistence.PersistentState; |
||
98 | import org.gvsig.tools.persistence.exception.PersistenceException; |
||
99 | import org.gvsig.tools.task.SimpleTaskStatus; |
||
100 | import org.gvsig.tools.task.TaskStatusManager; |
||
101 | 41059 | jjdelcerro | import org.gvsig.tools.visitor.VisitCanceledException; |
102 | import org.gvsig.tools.visitor.Visitor; |
||
103 | 40846 | jjdelcerro | import org.slf4j.Logger; |
104 | import org.slf4j.LoggerFactory; |
||
105 | 40878 | jjdelcerro | import org.supercsv.prefs.CsvPreference; |
106 | 44669 | jjdelcerro | import org.gvsig.tools.dataTypes.CoercionContext; |
107 | 47199 | jjdelcerro | import org.gvsig.tools.dispose.DisposeUtils; |
108 | import org.gvsig.tools.dynobject.DynObject; |
||
109 | 45929 | jjdelcerro | import org.gvsig.tools.i18n.I18nManager; |
110 | 45727 | jjdelcerro | import org.gvsig.tools.util.GetItemWithSize64; |
111 | 40846 | jjdelcerro | |
112 | 44309 | omartinez | @SuppressWarnings("UseSpecificCatch") |
113 | 47634 | fdiaz | public class SimpleReaderStoreProvider extends AbstractMemoryStoreProvider implements |
114 | 41617 | jjdelcerro | ResourceConsumer { |
115 | |||
116 | 47634 | fdiaz | private static final Logger LOGGER = LoggerFactory.getLogger(SimpleReaderStoreProvider.class); |
117 | 40846 | jjdelcerro | |
118 | 44669 | jjdelcerro | public static final String NAME = DataStore.CSV_PROVIDER_NAME; |
119 | 40846 | jjdelcerro | public static final String DESCRIPTION = "CSV file"; |
120 | |||
121 | public static final String METADATA_DEFINITION_NAME = NAME; |
||
122 | |||
123 | 45721 | jjdelcerro | private final ResourceProvider resource; |
124 | 40846 | jjdelcerro | |
125 | private long counterNewsOIDs = 0; |
||
126 | private Envelope envelope;
|
||
127 | 41059 | jjdelcerro | private boolean need_calculate_envelope = false; |
128 | 44309 | omartinez | private final SimpleTaskStatus taskStatus; |
129 | private final CSVFeatureWriter writer; |
||
130 | 45359 | omartinez | private FeatureType featureType;
|
131 | 45727 | jjdelcerro | private GetItemWithSize64<List<String>> virtualrows; |
132 | 45836 | fdiaz | private RowToFeatureTranslator rowToFeatureTranslator;
|
133 | 47199 | jjdelcerro | private SpatialIndex spatialIndex;
|
134 | |||
135 | 45721 | jjdelcerro | @SuppressWarnings({"OverridableMethodCallInConstructor", "LeakingThisInConstructor"}) |
136 | 47634 | fdiaz | public SimpleReaderStoreProvider(
|
137 | 47199 | jjdelcerro | CSVStoreParameters parameters, |
138 | DataStoreProviderServices storeServices |
||
139 | ) throws InitializeException {
|
||
140 | 40846 | jjdelcerro | super(
|
141 | 41617 | jjdelcerro | parameters, |
142 | storeServices, |
||
143 | FileHelper.newMetadataContainer(METADATA_DEFINITION_NAME) |
||
144 | 40846 | jjdelcerro | ); |
145 | 44309 | omartinez | this.writer = new CSVFeatureWriter(); |
146 | 40846 | jjdelcerro | TaskStatusManager manager = ToolsLocator.getTaskStatusManager(); |
147 | this.taskStatus = manager.createDefaultSimpleTaskStatus("CSV"); |
||
148 | 45836 | fdiaz | this.taskStatus.setAutoremove(true); |
149 | 40846 | jjdelcerro | |
150 | counterNewsOIDs = 0;
|
||
151 | |||
152 | File file = getCSVParameters().getFile();
|
||
153 | resource = this.createResource(
|
||
154 | 41617 | jjdelcerro | FileResource.NAME, |
155 | new Object[]{file.getAbsolutePath()} |
||
156 | 40846 | jjdelcerro | ); |
157 | |||
158 | resource.addConsumer(this);
|
||
159 | 44309 | omartinez | |
160 | 41006 | jjdelcerro | initializeFeatureTypes(); |
161 | 40846 | jjdelcerro | } |
162 | |||
163 | private CSVStoreParameters getCSVParameters() {
|
||
164 | return (CSVStoreParameters) this.getParameters(); |
||
165 | } |
||
166 | |||
167 | 44309 | omartinez | @Override
|
168 | 40846 | jjdelcerro | public String getProviderName() { |
169 | return NAME;
|
||
170 | } |
||
171 | |||
172 | 44309 | omartinez | @Override
|
173 | 40846 | jjdelcerro | public boolean allowWrite() { |
174 | 44309 | omartinez | return true; |
175 | 40846 | jjdelcerro | } |
176 | |||
177 | 41006 | jjdelcerro | private String getFullFileName() { |
178 | 41617 | jjdelcerro | // Usar solo para mostrar mensajes en el logger.
|
179 | 44309 | omartinez | String s;
|
180 | 41617 | jjdelcerro | try {
|
181 | s = getCSVParameters().getFile().getAbsolutePath(); |
||
182 | } catch (Exception e2) { |
||
183 | s = "(unknow)";
|
||
184 | } |
||
185 | return s;
|
||
186 | 41006 | jjdelcerro | } |
187 | 41617 | jjdelcerro | |
188 | 44309 | omartinez | @Override
|
189 | 40846 | jjdelcerro | public void open() throws OpenException { |
190 | 42775 | jjdelcerro | if (this.data != null) { |
191 | 40846 | jjdelcerro | return;
|
192 | } |
||
193 | 43215 | jjdelcerro | this.data = new ArrayList<>(); |
194 | 41006 | jjdelcerro | resource.setData(new HashMap()); |
195 | counterNewsOIDs = 0;
|
||
196 | 41617 | jjdelcerro | try {
|
197 | loadFeatures(); |
||
198 | } catch (RuntimeException e) { |
||
199 | 44408 | jjdelcerro | LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e); |
200 | 41617 | jjdelcerro | throw e;
|
201 | } catch (Exception e) { |
||
202 | 44408 | jjdelcerro | LOGGER.debug("Can't load features from CSV '" + getFullFileName() + "'.", e); |
203 | 41617 | jjdelcerro | throw new RuntimeException(e); |
204 | } |
||
205 | 40846 | jjdelcerro | } |
206 | |||
207 | 44309 | omartinez | @Override
|
208 | 40846 | jjdelcerro | public DataServerExplorer getExplorer() throws ReadException { |
209 | DataManager manager = DALLocator.getDataManager(); |
||
210 | 41006 | jjdelcerro | FilesystemServerExplorerParameters params; |
211 | 40846 | jjdelcerro | try {
|
212 | params = (FilesystemServerExplorerParameters) manager |
||
213 | 41617 | jjdelcerro | .createServerExplorerParameters(FilesystemServerExplorer.NAME); |
214 | 40846 | jjdelcerro | params.setRoot(this.getCSVParameters().getFile().getParent());
|
215 | 41617 | jjdelcerro | return manager.openServerExplorer(FilesystemServerExplorer.NAME, params);
|
216 | 45684 | jjdelcerro | } catch (Exception e) { |
217 | 40846 | jjdelcerro | throw new ReadException(this.getProviderName(), e); |
218 | } |
||
219 | |||
220 | } |
||
221 | |||
222 | 44309 | omartinez | @Override
|
223 | 45684 | jjdelcerro | @SuppressWarnings("Convert2Lambda") |
224 | 40846 | jjdelcerro | public void performChanges(Iterator deleteds, Iterator inserteds, Iterator updateds, Iterator originalFeatureTypesUpdated) throws PerformEditingException { |
225 | |||
226 | try {
|
||
227 | 45929 | jjdelcerro | I18nManager i18n = ToolsLocator.getI18nManager(); |
228 | 40846 | jjdelcerro | this.taskStatus.add();
|
229 | 45929 | jjdelcerro | taskStatus.message(i18n.getTranslation("_Preparing_writing"));
|
230 | 44309 | omartinez | this.getResource().closeRequest();
|
231 | 40846 | jjdelcerro | getResource().execute(new ResourceAction() {
|
232 | 44309 | omartinez | @Override
|
233 | 40846 | jjdelcerro | public Object run() throws Exception { |
234 | FeatureSet features = null;
|
||
235 | DisposableIterator it = null;
|
||
236 | try {
|
||
237 | File file = (File) resource.get(); |
||
238 | 45836 | fdiaz | features = getStoreServices().getFeatureStore().getFeatureSet(); |
239 | 41617 | jjdelcerro | |
240 | 45929 | jjdelcerro | taskStatus.message(i18n.getTranslation("_Writing_records"));
|
241 | 45836 | fdiaz | if (virtualrows == null) { |
242 | List<FeatureProvider> newdata = new ArrayList<>(); |
||
243 | FeatureType ftype = getStoreServices().getDefaultFeatureType(); |
||
244 | writer.initialize(getCSVParameters(), file, ftype, getCSVPreferences()); |
||
245 | writer.begin(); |
||
246 | it = features.fastIterator(); |
||
247 | 45929 | jjdelcerro | taskStatus.setRangeOfValues(0, features.getSize());
|
248 | taskStatus.setCurValue(0);
|
||
249 | 45836 | fdiaz | while (it.hasNext()) {
|
250 | 45929 | jjdelcerro | taskStatus.incrementCurrentValue(); |
251 | 45836 | fdiaz | FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature( |
252 | (org.gvsig.fmap.dal.feature.Feature) it.next()); |
||
253 | writer.add(feature); |
||
254 | if (feature.getOID() == null) { |
||
255 | LOGGER.warn("feature without OID");
|
||
256 | feature.setOID(createNewOID()); |
||
257 | } |
||
258 | newdata.add(feature); |
||
259 | 40846 | jjdelcerro | } |
260 | 45836 | fdiaz | data = newdata; |
261 | if (writer.getEnvelope() != null) { |
||
262 | 47199 | jjdelcerro | envelope = writer.getEnvelope(); |
263 | } else {
|
||
264 | envelope = null;
|
||
265 | 45836 | fdiaz | } |
266 | resource.notifyChanges(); |
||
267 | writer.end(); |
||
268 | 47199 | jjdelcerro | bboxFileSave(envelope); |
269 | 45836 | fdiaz | } else {
|
270 | |||
271 | CSVStoreParameters csvParams = getCSVParameters(); |
||
272 | CSVStoreParameters tmpParams = (CSVStoreParameters) csvParams.getCopy(); |
||
273 | |||
274 | String tmpBase = File.createTempFile("tmp_" + System.currentTimeMillis(), null).getAbsolutePath(); |
||
275 | File tmpFile = new File(tmpBase + ".csv"); |
||
276 | tmpParams.setFile(tmpFile); |
||
277 | |||
278 | FeatureType ftype = getStoreServices().getDefaultFeatureType(); |
||
279 | 47199 | jjdelcerro | writer.initialize(tmpParams, tmpFile, ftype, getCSVPreferences(),spatialIndex); |
280 | 45836 | fdiaz | writer.begin(); |
281 | it = features.fastIterator(); |
||
282 | 45929 | jjdelcerro | taskStatus.setIndeterminate(); |
283 | taskStatus.setCurValue(0);
|
||
284 | 45836 | fdiaz | while (it.hasNext()) {
|
285 | 45929 | jjdelcerro | taskStatus.incrementCurrentValue(); |
286 | if( taskStatus.isCancellationRequested() ) {
|
||
287 | taskStatus.cancel(); |
||
288 | LOGGER.info("CSV writing canceled ("+getFullFileName()+")"); |
||
289 | break;
|
||
290 | } |
||
291 | 45836 | fdiaz | FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature( |
292 | (org.gvsig.fmap.dal.feature.Feature) it.next()); |
||
293 | writer.add(feature); |
||
294 | if (feature.getOID() == null) { |
||
295 | LOGGER.warn("feature without OID");
|
||
296 | feature.setOID(createNewOID()); |
||
297 | } |
||
298 | } |
||
299 | if (writer.getEnvelope() != null) { |
||
300 | 47199 | jjdelcerro | envelope = writer.getEnvelope(); |
301 | } else {
|
||
302 | envelope = null;
|
||
303 | 45836 | fdiaz | } |
304 | resource.notifyChanges(); |
||
305 | writer.end(); |
||
306 | 45929 | jjdelcerro | if( !taskStatus.isCancelled() ) {
|
307 | if (!csvParams.getFile().delete()) {
|
||
308 | LOGGER.debug("Can't delete csv file '" + csvParams.getFile() + "'."); |
||
309 | throw new IOException("Can't delete csv '" |
||
310 | + FilenameUtils.getBaseName(csvParams.getFile().getAbsolutePath()) |
||
311 | + "' file to replace with the new csv.\nThe new csv is in temporary file '" + tmpFile.getAbsolutePath()
|
||
312 | + "'");
|
||
313 | } |
||
314 | 45836 | fdiaz | |
315 | 45929 | jjdelcerro | File csvFile = csvParams.getFile();
|
316 | FileUtils.moveFile(tmpParams.getFile(), csvFile); |
||
317 | FileUtils.delete(new File(FilenameUtils.removeExtension(csvFile.getAbsolutePath())+".idx")); |
||
318 | 45836 | fdiaz | |
319 | 47199 | jjdelcerro | bboxFileSave(envelope); |
320 | |||
321 | 45929 | jjdelcerro | loadFeatures(); |
322 | } |
||
323 | 40846 | jjdelcerro | } |
324 | } finally {
|
||
325 | 42775 | jjdelcerro | if (it != null) { |
326 | 40846 | jjdelcerro | it.dispose(); |
327 | } |
||
328 | 42775 | jjdelcerro | if (features != null) { |
329 | 40846 | jjdelcerro | features.dispose(); |
330 | } |
||
331 | } |
||
332 | return null; |
||
333 | } |
||
334 | 41617 | jjdelcerro | |
335 | 40846 | jjdelcerro | }); |
336 | this.taskStatus.terminate();
|
||
337 | } catch (Exception e) { |
||
338 | this.taskStatus.abort();
|
||
339 | throw new PerformEditingException(getResource().toString(), e); |
||
340 | } |
||
341 | } |
||
342 | |||
343 | 44309 | omartinez | private CsvPreference getCSVPreferences() {
|
344 | 45685 | jjdelcerro | CSVReaderSuperCSV reader = new CSVReaderSuperCSV(getCSVParameters());
|
345 | 44309 | omartinez | return reader.getCSVPreferences();
|
346 | } |
||
347 | |||
348 | @Override
|
||
349 | 40846 | jjdelcerro | public boolean closeResourceRequested(ResourceProvider resource) { |
350 | return true; |
||
351 | } |
||
352 | |||
353 | 44309 | omartinez | @Override
|
354 | 40846 | jjdelcerro | public int getOIDType() { |
355 | return DataTypes.LONG;
|
||
356 | } |
||
357 | |||
358 | 44309 | omartinez | @Override
|
359 | 40846 | jjdelcerro | public boolean supportsAppendMode() { |
360 | 44309 | omartinez | return true; |
361 | 40846 | jjdelcerro | } |
362 | |||
363 | 44309 | omartinez | @Override
|
364 | 45684 | jjdelcerro | @SuppressWarnings("Convert2Lambda") |
365 | 44309 | omartinez | public void append(final FeatureProvider featureProvider) { |
366 | //todo
|
||
367 | getResource().execute(new ResourceAction() {
|
||
368 | @Override
|
||
369 | public Object run() throws Exception { |
||
370 | //writer.append(getStoreServices().createFeature(featureProvider));
|
||
371 | writer.add(featureProvider); |
||
372 | return null; |
||
373 | } |
||
374 | }); |
||
375 | 40846 | jjdelcerro | } |
376 | |||
377 | 44309 | omartinez | @Override
|
378 | 45684 | jjdelcerro | @SuppressWarnings("Convert2Lambda") |
379 | 44309 | omartinez | public void beginAppend() throws DataException { |
380 | getResource().execute(new ResourceAction() {
|
||
381 | 45684 | jjdelcerro | @Override
|
382 | 44309 | omartinez | public Object run() throws Exception { |
383 | writer.initialize( |
||
384 | getCSVParameters(), |
||
385 | getCSVParameters().getFile(), |
||
386 | getFeatureStore().getDefaultFeatureType(), |
||
387 | 44388 | omartinez | getCSVPreferences() |
388 | 44309 | omartinez | ); |
389 | writer.beginAppend(); |
||
390 | return null; |
||
391 | } |
||
392 | }); |
||
393 | |||
394 | 40846 | jjdelcerro | } |
395 | |||
396 | 44309 | omartinez | @Override
|
397 | 45684 | jjdelcerro | @SuppressWarnings("Convert2Lambda") |
398 | 40846 | jjdelcerro | public void endAppend() { |
399 | 44309 | omartinez | try {
|
400 | getResource().execute(new ResourceAction() {
|
||
401 | 45684 | jjdelcerro | @Override
|
402 | 44309 | omartinez | public Object run() throws Exception { |
403 | writer.end(); |
||
404 | resource.notifyChanges(); //resourcesNotifyChanges();
|
||
405 | counterNewsOIDs = -1;
|
||
406 | return null; |
||
407 | } |
||
408 | }); |
||
409 | 47199 | jjdelcerro | if (writer.getEnvelope() != null) { |
410 | envelope = writer.getEnvelope(); |
||
411 | } else {
|
||
412 | envelope = null;
|
||
413 | } |
||
414 | 44309 | omartinez | writer.end(); |
415 | 46608 | fdiaz | this.close();
|
416 | 47199 | jjdelcerro | bboxFileSave(envelope); |
417 | |||
418 | 46608 | fdiaz | } catch (Exception ex) { |
419 | 44408 | jjdelcerro | LOGGER.warn("Not been able to end append '" + this.getFullName() + "'.", ex); |
420 | 44309 | omartinez | } |
421 | 40846 | jjdelcerro | } |
422 | |||
423 | public void saveToState(PersistentState state) throws PersistenceException { |
||
424 | throw new NotYetImplemented(); |
||
425 | } |
||
426 | |||
427 | public void loadFromState(PersistentState state) throws PersistenceException { |
||
428 | throw new NotYetImplemented(); |
||
429 | } |
||
430 | |||
431 | 44309 | omartinez | @Override
|
432 | 40846 | jjdelcerro | public Object createNewOID() { |
433 | 44309 | omartinez | return counterNewsOIDs++;
|
434 | 40846 | jjdelcerro | } |
435 | |||
436 | protected void initializeFeatureTypes() throws InitializeException { |
||
437 | try {
|
||
438 | this.open();
|
||
439 | } catch (OpenException e) {
|
||
440 | throw new InitializeException(this.getProviderName(), e); |
||
441 | } |
||
442 | } |
||
443 | |||
444 | 44309 | omartinez | @Override
|
445 | 45684 | jjdelcerro | @SuppressWarnings("Convert2Lambda") |
446 | 40846 | jjdelcerro | public Envelope getEnvelope() throws DataException { |
447 | this.open();
|
||
448 | 42775 | jjdelcerro | if (this.envelope != null) { |
449 | 41617 | jjdelcerro | return this.envelope; |
450 | 41059 | jjdelcerro | } |
451 | 47199 | jjdelcerro | this.envelope = bboxFileLoad();
|
452 | if (this.envelope != null) { |
||
453 | return this.envelope; |
||
454 | } |
||
455 | 42775 | jjdelcerro | if (!this.need_calculate_envelope) { |
456 | 41617 | jjdelcerro | return null; |
457 | 41059 | jjdelcerro | } |
458 | try {
|
||
459 | 45929 | jjdelcerro | I18nManager i18n = ToolsLocator.getI18nManager(); |
460 | 45836 | fdiaz | this.taskStatus.add();
|
461 | 45929 | jjdelcerro | this.taskStatus.message(i18n.getTranslation("_Calculating_envelope")); |
462 | 45836 | fdiaz | FeatureStore fs = this.getFeatureStore();
|
463 | FeatureType ft = fs.getDefaultFeatureType(); |
||
464 | FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex()); |
||
465 | this.taskStatus.setRangeOfValues(0, fs.getFeatureCount()); |
||
466 | 41059 | jjdelcerro | this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType());
|
467 | 41617 | jjdelcerro | fs.accept(new Visitor() {
|
468 | 44309 | omartinez | @Override
|
469 | 41617 | jjdelcerro | public void visit(Object obj) throws VisitCanceledException, BaseException { |
470 | 45836 | fdiaz | taskStatus.incrementCurrentValue(); |
471 | if(taskStatus.isCancellationRequested()){
|
||
472 | taskStatus.cancel(); |
||
473 | throw new VisitCanceledException(); |
||
474 | } |
||
475 | 41617 | jjdelcerro | Feature f = (Feature) obj; |
476 | Geometry geom = f.getDefaultGeometry(); |
||
477 | 42775 | jjdelcerro | if (geom != null) { |
478 | 47199 | jjdelcerro | try {
|
479 | Envelope env = geom.getEnvelope(); |
||
480 | envelope.add(env); |
||
481 | } catch(Exception ex) { |
||
482 | LOGGER.warn("Can't calculate envelop of geometry in feature '"+Objects.toString(f.getReference())+"'.",ex); |
||
483 | } |
||
484 | 41617 | jjdelcerro | } |
485 | } |
||
486 | }); |
||
487 | 47199 | jjdelcerro | bboxFileSave(envelope); |
488 | 45836 | fdiaz | taskStatus.terminate(); |
489 | } catch (VisitCanceledException e) {
|
||
490 | return null; |
||
491 | 41617 | jjdelcerro | } catch (BaseException e) {
|
492 | 45836 | fdiaz | taskStatus.abort(); |
493 | 44408 | jjdelcerro | LOGGER.warn("Can't calculate the envelope of CSV file '" + this.getFullName() + "'.", e); |
494 | 45836 | fdiaz | return null; |
495 | 41617 | jjdelcerro | } |
496 | |||
497 | 41059 | jjdelcerro | this.need_calculate_envelope = false; |
498 | 40846 | jjdelcerro | return this.envelope; |
499 | } |
||
500 | |||
501 | 44309 | omartinez | @Override
|
502 | 40846 | jjdelcerro | public Object getDynValue(String name) throws DynFieldNotFoundException { |
503 | 42775 | jjdelcerro | if (DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name)) {
|
504 | 40846 | jjdelcerro | try {
|
505 | return this.getEnvelope(); |
||
506 | } catch (DataException e) {
|
||
507 | return null; |
||
508 | } |
||
509 | } else {
|
||
510 | 42775 | jjdelcerro | if (DataStore.METADATA_CRS.equalsIgnoreCase(name)) {
|
511 | 40878 | jjdelcerro | IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters());
|
512 | 42775 | jjdelcerro | if (pro != null) { |
513 | 40846 | jjdelcerro | return pro;
|
514 | } |
||
515 | } |
||
516 | } |
||
517 | return super.getDynValue(name); |
||
518 | } |
||
519 | |||
520 | 44309 | omartinez | @Override
|
521 | 40846 | jjdelcerro | public void resourceChanged(ResourceProvider resource) { |
522 | this.getStoreServices().notifyChange(
|
||
523 | 41617 | jjdelcerro | DataStoreNotification.RESOURCE_CHANGED, |
524 | resource); |
||
525 | 40846 | jjdelcerro | } |
526 | |||
527 | 44309 | omartinez | @Override
|
528 | 40846 | jjdelcerro | public Object getSourceId() { |
529 | return this.getCSVParameters().getFile(); |
||
530 | } |
||
531 | |||
532 | 44309 | omartinez | @Override
|
533 | 40846 | jjdelcerro | public String getName() { |
534 | 40878 | jjdelcerro | String name = this.getCSVParameters().getFile().getName(); |
535 | 41006 | jjdelcerro | return FilenameUtils.getBaseName(name);
|
536 | 40846 | jjdelcerro | } |
537 | |||
538 | 44309 | omartinez | @Override
|
539 | 40846 | jjdelcerro | public String getFullName() { |
540 | return this.getCSVParameters().getFile().getAbsolutePath(); |
||
541 | } |
||
542 | |||
543 | 44309 | omartinez | @Override
|
544 | 40846 | jjdelcerro | public ResourceProvider getResource() {
|
545 | return resource;
|
||
546 | } |
||
547 | |||
548 | private boolean isEmpty(String s) { |
||
549 | 42775 | jjdelcerro | if (s == null) { |
550 | 41617 | jjdelcerro | return true; |
551 | } |
||
552 | return s.trim().length() == 0; |
||
553 | 40846 | jjdelcerro | } |
554 | 40878 | jjdelcerro | |
555 | 44309 | omartinez | private void init(CSVStoreParameters parameters, DataStoreProviderServices storeServices) { |
556 | throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. |
||
557 | } |
||
558 | 41617 | jjdelcerro | |
559 | 41006 | jjdelcerro | static class ToPointEvaluaror extends AbstractEvaluator { |
560 | 40846 | jjdelcerro | |
561 | 41335 | jjdelcerro | private static final Logger logger = LoggerFactory.getLogger(ToPointEvaluaror.class); |
562 | 40846 | jjdelcerro | |
563 | 41335 | jjdelcerro | private GeometryManager geommgr = null; |
564 | private String xname = null; |
||
565 | private String yname = null; |
||
566 | private String zname = null; |
||
567 | 45684 | jjdelcerro | private final Coercion toDouble; |
568 | 41335 | jjdelcerro | private int errorcount = 0; |
569 | |||
570 | ToPointEvaluaror(String[] pointDimensionNames) { |
||
571 | this.xname = pointDimensionNames[0]; |
||
572 | this.yname = pointDimensionNames[1]; |
||
573 | 42775 | jjdelcerro | if (pointDimensionNames.length > 2) { |
574 | 41335 | jjdelcerro | this.zname = pointDimensionNames[2]; |
575 | } |
||
576 | this.geommgr = GeometryLocator.getGeometryManager();
|
||
577 | this.toDouble = ToolsLocator.getDataTypesManager().getCoercion(DataTypes.DOUBLE);
|
||
578 | } |
||
579 | |||
580 | 44309 | omartinez | @Override
|
581 | 41335 | jjdelcerro | public Object evaluate(EvaluatorData data) throws EvaluatorException { |
582 | try {
|
||
583 | 45684 | jjdelcerro | double x = ((Double) toDouble.coerce(data.getDataValue(xname))); |
584 | double y = ((Double) toDouble.coerce(data.getDataValue(yname))); |
||
585 | 41335 | jjdelcerro | Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D);
|
586 | 42775 | jjdelcerro | if (zname != null) { |
587 | 45684 | jjdelcerro | double z = ((Double) toDouble.coerce(data.getDataValue(zname))); |
588 | 41335 | jjdelcerro | point.setCoordinateAt(2, z);
|
589 | } |
||
590 | return point;
|
||
591 | } catch (Exception ex) { |
||
592 | 42775 | jjdelcerro | if (++errorcount < 5) { |
593 | 41335 | jjdelcerro | logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='" |
594 | + xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString()); |
||
595 | } |
||
596 | return null; |
||
597 | } |
||
598 | } |
||
599 | |||
600 | 44309 | omartinez | @Override
|
601 | 41335 | jjdelcerro | public String getName() { |
602 | return "ToPointEvaluaror"; |
||
603 | } |
||
604 | |||
605 | 40846 | jjdelcerro | } |
606 | 41335 | jjdelcerro | |
607 | 45727 | jjdelcerro | public static class RowToFeatureTranslator { |
608 | 45836 | fdiaz | |
609 | 45727 | jjdelcerro | private Coercion coercion[]; |
610 | private CoercionContext coercionContext[]; |
||
611 | private int sizes[]; |
||
612 | 45990 | jjdelcerro | private String[] names; |
613 | 45727 | jjdelcerro | private final boolean ignore_errors; |
614 | private long count_errors; |
||
615 | 45836 | fdiaz | private FeatureType csvFeatureType;
|
616 | 45727 | jjdelcerro | |
617 | public RowToFeatureTranslator(boolean ignore_errors) { |
||
618 | this.ignore_errors = ignore_errors;
|
||
619 | this.count_errors = 0; |
||
620 | } |
||
621 | |||
622 | public int getColumnSize(int column) { |
||
623 | return this.sizes[column]; |
||
624 | } |
||
625 | 45836 | fdiaz | |
626 | 45727 | jjdelcerro | public void initialize(FeatureType ftype) { |
627 | 45836 | fdiaz | this.csvFeatureType = ftype;
|
628 | int columns = this.csvFeatureType.size(); |
||
629 | 45990 | jjdelcerro | this.names = new String[columns]; |
630 | 45727 | jjdelcerro | this.coercion = new Coercion[columns]; |
631 | this.coercionContext = new CoercionContext[columns]; |
||
632 | this.sizes = new int[columns]; |
||
633 | 45990 | jjdelcerro | int index = 0; |
634 | 45836 | fdiaz | for (int i = 0; i < this.csvFeatureType.size(); i++) { |
635 | FeatureAttributeDescriptor ad = this.csvFeatureType.getAttributeDescriptor(i);
|
||
636 | 45990 | jjdelcerro | names[i] = null;
|
637 | if( ad.isComputed() ) {
|
||
638 | continue;
|
||
639 | } |
||
640 | names[index] = ad.getName(); |
||
641 | coercion[index] = ad.getCoercion(); |
||
642 | coercionContext[index] = ad.getCoercionContext(); |
||
643 | sizes[index] = ad.getSize(); |
||
644 | index++; |
||
645 | 45727 | jjdelcerro | } |
646 | } |
||
647 | 45836 | fdiaz | |
648 | 45727 | jjdelcerro | public void translate(long rowindex, List<String> row, FeatureProvider feature) throws Exception { |
649 | |||
650 | 45836 | fdiaz | feature.setOID(rowindex); |
651 | 45990 | jjdelcerro | for (int i = 0; i < names.length; i++) { |
652 | String name = names[i];
|
||
653 | if( name == null ) { |
||
654 | break;
|
||
655 | } |
||
656 | Object rawvalue = row.get(i);
|
||
657 | 45727 | jjdelcerro | try {
|
658 | Object value = null; |
||
659 | 45990 | jjdelcerro | if (coercion[i] != null) { |
660 | value = coercion[i].coerce(rawvalue, coercionContext[i]); |
||
661 | 45727 | jjdelcerro | } |
662 | 46080 | jjdelcerro | int findex = feature.getType().getIndex(name);
|
663 | if( findex>=0 ) { |
||
664 | // Ojo que puede que se este filtrando el featuretype y no
|
||
665 | // tenga todos los atributos, por ejemplo al pintar la vista.
|
||
666 | feature.set(findex, value); |
||
667 | } |
||
668 | 45990 | jjdelcerro | if (sizes[i] >= 0 |
669 | 45727 | jjdelcerro | && (value instanceof String || value instanceof URL |
670 | || value instanceof URI || value instanceof File)) { |
||
671 | int x = value.toString().length();
|
||
672 | 45990 | jjdelcerro | if (sizes[i] < x) {
|
673 | sizes[i] = x; |
||
674 | 45727 | jjdelcerro | } |
675 | } |
||
676 | } catch (Exception ex) { |
||
677 | if (!ignore_errors) {
|
||
678 | throw ex;
|
||
679 | } |
||
680 | if (count_errors++ < 10) { |
||
681 | 45990 | jjdelcerro | LOGGER.warn("Can't load value of attribute " + name +"/" +i+" in row " + rowindex + ".", ex); |
682 | 45727 | jjdelcerro | } |
683 | if (count_errors == 10) { |
||
684 | LOGGER.info("Too many errors, suppress messages.");
|
||
685 | } |
||
686 | } |
||
687 | } |
||
688 | } |
||
689 | } |
||
690 | |||
691 | 45684 | jjdelcerro | private void loadFeatures() { |
692 | InputStreamReader in = null; |
||
693 | SimpleReader reader = null;
|
||
694 | try {
|
||
695 | 45929 | jjdelcerro | taskStatus.setTitle("CSV "+this.getName()); |
696 | 45817 | jjdelcerro | taskStatus.add(); |
697 | 45727 | jjdelcerro | // boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
|
698 | 45684 | jjdelcerro | |
699 | // Initialize the feature types
|
||
700 | EditableFeatureType edftype = getStoreServices().createFeatureType(this.getName());
|
||
701 | 45929 | jjdelcerro | CSVUtils.loadFeatureType(getCSVParameters(), edftype, true, taskStatus);
|
702 | 41617 | jjdelcerro | FeatureType ftype = edftype.getNotEditableCopy(); |
703 | 45359 | omartinez | this.setFeatureType(ftype);
|
704 | 41059 | jjdelcerro | |
705 | 45685 | jjdelcerro | in = CSVUtils.openFile( |
706 | 45684 | jjdelcerro | this.getCSVParameters().getFile(),
|
707 | CSVStoreParameters.getCharset(this.getCSVParameters())
|
||
708 | ); |
||
709 | 47634 | fdiaz | reader = getSimpleReader(getCSVParameters(), in); |
710 | 45684 | jjdelcerro | if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
711 | reader.getHeader(); // Skip and ignore the header of file
|
||
712 | } |
||
713 | 45727 | jjdelcerro | this.rowToFeatureTranslator = new RowToFeatureTranslator( |
714 | CSVStoreParameters.getIgnoreErrors(getCSVParameters()) |
||
715 | ); |
||
716 | this.rowToFeatureTranslator.initialize(ftype);
|
||
717 | 42775 | jjdelcerro | if (ftype.getDefaultGeometryAttributeName() != null) { |
718 | 41617 | jjdelcerro | this.need_calculate_envelope = true; |
719 | } |
||
720 | 45929 | jjdelcerro | I18nManager i18n = ToolsLocator.getI18nManager(); |
721 | taskStatus.message(i18n.getTranslation("_Loading"));
|
||
722 | 41006 | jjdelcerro | |
723 | 45836 | fdiaz | if(this.virtualrows != null && this.virtualrows instanceof Closeable){ |
724 | IOUtils.closeQuietly((Closeable) this.virtualrows); |
||
725 | this.virtualrows = null; |
||
726 | } |
||
727 | |||
728 | this.virtualrows = ((AbstractSimpleReader) reader).getVirtualRows(this.taskStatus); |
||
729 | if (this.virtualrows == null) { |
||
730 | 41062 | jjdelcerro | |
731 | 45727 | jjdelcerro | List<String> row = reader.read(); |
732 | 41062 | jjdelcerro | |
733 | 45727 | jjdelcerro | int skipLines = CSVStoreParameters.getSkipLines(getCSVParameters());
|
734 | if (skipLines > 0) { |
||
735 | row = reader.skip(skipLines); |
||
736 | } |
||
737 | int limit = CSVStoreParameters.getLimit(getCSVParameters());
|
||
738 | while (row != null) { |
||
739 | 45929 | jjdelcerro | taskStatus.incrementCurrentValue(); |
740 | if( taskStatus.isCancellationRequested() ) {
|
||
741 | taskStatus.cancel(); |
||
742 | break;
|
||
743 | } |
||
744 | 45727 | jjdelcerro | FeatureProvider feature = this.createFeatureProvider(ftype);
|
745 | 45929 | jjdelcerro | this.rowToFeatureTranslator.translate(reader.getLine(), row, feature);
|
746 | 45836 | fdiaz | |
747 | 45727 | jjdelcerro | this.addFeatureProvider(feature);
|
748 | if (limit > 0) { |
||
749 | if (limit < this.data.size()) { |
||
750 | break;
|
||
751 | 44749 | jjdelcerro | } |
752 | 41617 | jjdelcerro | } |
753 | 45727 | jjdelcerro | row = reader.read(); |
754 | 41617 | jjdelcerro | } |
755 | 45727 | jjdelcerro | for (int i = 0; i < ftype.size(); i++) { |
756 | 45836 | fdiaz | if (this.rowToFeatureTranslator.getColumnSize(i) > 0) { |
757 | 45727 | jjdelcerro | // if (sizes[i] > 0) {
|
758 | EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i)); |
||
759 | // efad.setSize(sizes[i]);
|
||
760 | efad.setSize(this.rowToFeatureTranslator.getColumnSize(i));
|
||
761 | 42775 | jjdelcerro | } |
762 | } |
||
763 | 45727 | jjdelcerro | // Volvemos a asignar al store el featuretype, ya que puede
|
764 | // haber cambiado.
|
||
765 | ftype = edftype.getNotEditableCopy(); |
||
766 | this.setFeatureType(ftype);
|
||
767 | 47199 | jjdelcerro | } else {
|
768 | this.loadOrCreateSpatialIndex();
|
||
769 | 41617 | jjdelcerro | } |
770 | 45929 | jjdelcerro | if( taskStatus.isRunning() ) {
|
771 | taskStatus.terminate(); |
||
772 | } |
||
773 | 44963 | jjdelcerro | } catch (Throwable ex) { |
774 | 45817 | jjdelcerro | taskStatus.abort(); |
775 | 41643 | jjdelcerro | int lineno = 0; |
776 | 42775 | jjdelcerro | if (reader != null) { |
777 | 41643 | jjdelcerro | lineno = reader.getLine(); |
778 | } |
||
779 | 42775 | jjdelcerro | throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex); |
780 | 41643 | jjdelcerro | |
781 | 41617 | jjdelcerro | } finally {
|
782 | 42775 | jjdelcerro | if (reader != null) { |
783 | 41617 | jjdelcerro | try {
|
784 | reader.close(); |
||
785 | } catch (Exception ex) { |
||
786 | // Do nothing
|
||
787 | } |
||
788 | 45684 | jjdelcerro | // reader = null;
|
789 | 41617 | jjdelcerro | } |
790 | 42775 | jjdelcerro | if (in != null) { |
791 | 41617 | jjdelcerro | try {
|
792 | in.close(); |
||
793 | } catch (Exception ex) { |
||
794 | // Do nothing
|
||
795 | } |
||
796 | 45684 | jjdelcerro | // in = null;
|
797 | 41617 | jjdelcerro | } |
798 | 47199 | jjdelcerro | taskStatus.remove(); |
799 | 41617 | jjdelcerro | } |
800 | } |
||
801 | 41062 | jjdelcerro | |
802 | 45359 | omartinez | @Override
|
803 | public void fixFeatureTypeFromParameters() { |
||
804 | 46608 | fdiaz | if(mustFixFeatureType() && featureType != null){ |
805 | this.setFeatureType(featureType);
|
||
806 | } |
||
807 | } |
||
808 | |||
809 | private boolean mustFixFeatureType() { |
||
810 | 46568 | fdiaz | try {
|
811 | String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters()); |
||
812 | String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters()); |
||
813 | String geometry_column = CSVStoreParameters.getGeometryColumn(this.getParameters()); |
||
814 | |||
815 | |||
816 | FeatureType dalFeatureType = this.getStoreServices().getDefaultFeatureType();
|
||
817 | if (StringUtils.isNotBlank(geometry_column)){
|
||
818 | FeatureAttributeDescriptor attr = dalFeatureType.getAttributeDescriptor(geometry_column); |
||
819 | if(attr == null || attr.getType() != DataTypes.GEOMETRY){ |
||
820 | 46608 | fdiaz | return true; |
821 | 46568 | fdiaz | } |
822 | } |
||
823 | if (StringUtils.isNotBlank(param_types_def)
|
||
824 | || pointDimensionNames != null) {
|
||
825 | for (FeatureAttributeDescriptor dalAttr : dalFeatureType) {
|
||
826 | if(dalAttr.isComputed()){
|
||
827 | continue;
|
||
828 | } |
||
829 | FeatureAttributeDescriptor csvAttr = featureType.getAttributeDescriptor(dalAttr.getName()); |
||
830 | if(csvAttr == null){ |
||
831 | 46608 | fdiaz | return true; |
832 | 46568 | fdiaz | } |
833 | if(!dalAttr.get("all").equals(csvAttr.get("all"))){ |
||
834 | 46608 | fdiaz | return true; |
835 | 46568 | fdiaz | } |
836 | } |
||
837 | } |
||
838 | } catch (DataException ex) {
|
||
839 | 46608 | fdiaz | LOGGER.warn("Can't check if should fix the feature type", ex);
|
840 | 45359 | omartinez | } |
841 | 46608 | fdiaz | return false; |
842 | 45359 | omartinez | } |
843 | 45836 | fdiaz | |
844 | 45359 | omartinez | private void setFeatureType(FeatureType ftype) { |
845 | 46608 | fdiaz | try {
|
846 | List<FeatureType> ftypes = new ArrayList<>(); |
||
847 | ftypes.add(ftype); |
||
848 | this.featureType = ftype;
|
||
849 | if(this.getStoreServices().getDefaultFeatureType() == null){ |
||
850 | this.getStoreServices().setFeatureTypes(ftypes, ftype);
|
||
851 | return;
|
||
852 | } |
||
853 | if(mustFixFeatureType()){
|
||
854 | this.getStoreServices().setFeatureTypes(ftypes, ftype);
|
||
855 | } |
||
856 | } catch (DataException ex) {
|
||
857 | LOGGER.warn("Cant set feature type", ex);
|
||
858 | } |
||
859 | 45359 | omartinez | } |
860 | 42775 | jjdelcerro | |
861 | 45727 | jjdelcerro | @Override
|
862 | 47557 | fdiaz | public FeatureSetProvider createSet(FeatureQuery query, FeatureType providerFeatureType, FeatureType featureType)
|
863 | throws DataException {
|
||
864 | this.open();
|
||
865 | if (this.virtualrows == null) { |
||
866 | return super.createSet(query, providerFeatureType, featureType); |
||
867 | } |
||
868 | 47634 | fdiaz | return new SimpleReaderSetProvider(this, query, providerFeatureType, featureType); |
869 | 47557 | fdiaz | } |
870 | |||
871 | @Override
|
||
872 | 45727 | jjdelcerro | public FeatureSetProvider createSet(FeatureQuery query, FeatureType featureType)
|
873 | throws DataException {
|
||
874 | 47199 | jjdelcerro | this.open();
|
875 | 45836 | fdiaz | if (this.virtualrows == null) { |
876 | 45727 | jjdelcerro | return super.createSet(query, featureType); |
877 | } |
||
878 | 47634 | fdiaz | return new SimpleReaderSetProvider(this, query, featureType, featureType); |
879 | 45836 | fdiaz | } |
880 | 45727 | jjdelcerro | |
881 | public List<String> getRowByIndex(long index) { |
||
882 | 47199 | jjdelcerro | try {
|
883 | this.open();
|
||
884 | } catch(Exception ex) { |
||
885 | throw new RuntimeException("Can't get row by index", ex); |
||
886 | } |
||
887 | 45836 | fdiaz | if (this.virtualrows == null) { |
888 | 45727 | jjdelcerro | return null; |
889 | } |
||
890 | 46136 | jjdelcerro | List<String> line = this.virtualrows.get64(index); |
891 | if( line!=null ) { |
||
892 | for (int i = 0; i < line.size(); i++) { |
||
893 | String s = line.get(i);
|
||
894 | 46161 | jjdelcerro | line.set(i, CSVReaderSuperCSV.unescapeCRLF(s)); |
895 | 46136 | jjdelcerro | } |
896 | } |
||
897 | return line;
|
||
898 | 45727 | jjdelcerro | } |
899 | 45836 | fdiaz | |
900 | 45727 | jjdelcerro | public RowToFeatureTranslator getRowToFeatureTranslator() {
|
901 | 45836 | fdiaz | if (this.rowToFeatureTranslator == null) { |
902 | 45727 | jjdelcerro | boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters());
|
903 | this.rowToFeatureTranslator = new RowToFeatureTranslator(ignore_errors); |
||
904 | this.rowToFeatureTranslator.initialize(featureType);
|
||
905 | } |
||
906 | return this.rowToFeatureTranslator; |
||
907 | } |
||
908 | |||
909 | @Override
|
||
910 | public long getFeatureCount() throws DataException { |
||
911 | 47199 | jjdelcerro | this.open();
|
912 | 45836 | fdiaz | if (this.virtualrows == null) { |
913 | return super.getFeatureCount(); |
||
914 | 45727 | jjdelcerro | } |
915 | return this.virtualrows.size64(); |
||
916 | } |
||
917 | 45836 | fdiaz | |
918 | 45929 | jjdelcerro | @Override
|
919 | 45836 | fdiaz | public long getDataSize() throws DataException { |
920 | 47199 | jjdelcerro | this.open();
|
921 | 45836 | fdiaz | if (this.virtualrows == null) { |
922 | return super.getDataSize(); |
||
923 | } |
||
924 | return this.virtualrows.size64(); |
||
925 | } |
||
926 | |||
927 | @Override
|
||
928 | protected FeatureProvider internalGetFeatureProviderByReference(
|
||
929 | FeatureReferenceProviderServices reference) throws DataException {
|
||
930 | 47199 | jjdelcerro | this.open();
|
931 | 45836 | fdiaz | if (this.virtualrows == null) { |
932 | return super.internalGetFeatureProviderByReference(reference); |
||
933 | } |
||
934 | int oid = ((Long) reference.getOID()).intValue(); |
||
935 | RowToFeatureTranslator translator = getRowToFeatureTranslator(); |
||
936 | FeatureProvider feature = this.createFeatureProvider(this.featureType); |
||
937 | try {
|
||
938 | translator.translate(oid, this.virtualrows.get64(oid), feature);
|
||
939 | } catch (Exception ex) { |
||
940 | throw new CreateFeatureException(ex, this.getName()); |
||
941 | } |
||
942 | return feature;
|
||
943 | } |
||
944 | |||
945 | @Override
|
||
946 | protected void doDispose() throws BaseException { |
||
947 | super.doDispose();
|
||
948 | if (this.virtualrows != null && this.virtualrows instanceof Closeable) { |
||
949 | IOUtils.closeQuietly((Closeable) this.virtualrows); |
||
950 | this.virtualrows = null; |
||
951 | } |
||
952 | } |
||
953 | 45727 | jjdelcerro | |
954 | 47199 | jjdelcerro | @Override
|
955 | 46608 | fdiaz | public void refresh() throws OpenException { |
956 | try {
|
||
957 | this.close();
|
||
958 | } catch (CloseException e) {
|
||
959 | throw new OpenException(this.getProviderName(), e); |
||
960 | } |
||
961 | this.open();
|
||
962 | } |
||
963 | |||
964 | @Override
|
||
965 | public void close() throws CloseException { |
||
966 | super.close(); //To change body of generated methods, choose Tools | Templates. |
||
967 | this.data = null; |
||
968 | 46614 | fdiaz | if(this.virtualrows != null && this.virtualrows instanceof Closeable){ |
969 | IOUtils.closeQuietly((Closeable) this.virtualrows); |
||
970 | this.virtualrows = null; |
||
971 | 47199 | jjdelcerro | this.envelope = null; |
972 | this.spatialIndex = null; |
||
973 | 46614 | fdiaz | } |
974 | |||
975 | 46608 | fdiaz | } |
976 | |||
977 | 45727 | jjdelcerro | |
978 | 47199 | jjdelcerro | private void loadOrCreateSpatialIndex() { |
979 | FeatureSetProvider set = null;
|
||
980 | DisposableIterator<FeatureProvider> it = null;
|
||
981 | try {
|
||
982 | if( this.virtualrows == null ) { |
||
983 | return;
|
||
984 | } |
||
985 | FeatureAttributeDescriptor geomdesc = this.featureType.getDefaultGeometryAttribute();
|
||
986 | if( geomdesc == null ) { |
||
987 | return;
|
||
988 | } |
||
989 | // String indexTypeName = "MVRTree";
|
||
990 | // String extname = "mvtree";
|
||
991 | String indexTypeName = GeometryManager.SPATIALINDEX_DEFAULT_QUADTREE;
|
||
992 | String extname = "qtree"; |
||
993 | |||
994 | this.envelope = bboxFileLoad();
|
||
995 | File indexfile = this.getAuxFile(extname); |
||
996 | boolean createIndex = !indexfile.exists();
|
||
997 | 45836 | fdiaz | |
998 | 47199 | jjdelcerro | GeometryManager geomManager = GeometryLocator.getGeometryManager(); |
999 | SpatialIndexFactory indexfactory = geomManager.getSpatialIndexFactory(indexTypeName); |
||
1000 | DynObject params = indexfactory.createParameters(); |
||
1001 | params.setDynValue("file", indexfile);
|
||
1002 | SpatialIndex index = geomManager.createSpatialIndex(indexTypeName, params); |
||
1003 | if( createIndex ) {
|
||
1004 | I18nManager i18n = ToolsLocator.getI18nManager(); |
||
1005 | this.taskStatus.add();
|
||
1006 | taskStatus.message(i18n.getTranslation("_Creating_spatial_index"));
|
||
1007 | taskStatus.setRangeOfValues(0, this.virtualrows.size64()); |
||
1008 | taskStatus.setCurValue(0);
|
||
1009 | Envelope theEnvelope = geomManager.createEnvelope(Geometry.SUBTYPES.GEOM2D); |
||
1010 | set = this.createSet(null, featureType); |
||
1011 | it = set.fastIterator(); |
||
1012 | while( it.hasNext() ) {
|
||
1013 | taskStatus.incrementCurrentValue(); |
||
1014 | if( taskStatus.isCancellationRequested() ) {
|
||
1015 | taskStatus.cancel(); |
||
1016 | LOGGER.info("CSV spatial index creation canceled ("+getFullFileName()+")"); |
||
1017 | break;
|
||
1018 | } |
||
1019 | FeatureProvider f = it.next(); |
||
1020 | if( f == null ) { |
||
1021 | continue;
|
||
1022 | } |
||
1023 | Object oid = null; |
||
1024 | try {
|
||
1025 | oid = f.getOID(); |
||
1026 | Geometry geom = (Geometry) f.get(geomdesc.getName()); |
||
1027 | index.insert(geom, oid); |
||
1028 | theEnvelope.add(geom); |
||
1029 | } catch(Exception ex) { |
||
1030 | LOGGER.debug("Can't insert feature '"+Objects.toString(oid)+"' in spatial index.",ex); |
||
1031 | } |
||
1032 | } |
||
1033 | taskStatus.message(i18n.getTranslation("_Saving_spatial_index"));
|
||
1034 | taskStatus.setIndeterminate(); |
||
1035 | index.flush(); |
||
1036 | bboxFileSave(theEnvelope); |
||
1037 | taskStatus.terminate(); |
||
1038 | this.envelope = theEnvelope;
|
||
1039 | } |
||
1040 | this.spatialIndex = index;
|
||
1041 | } catch (Exception ex) { |
||
1042 | taskStatus.abort(); |
||
1043 | LOGGER.warn("Can't create spatial index.",ex);
|
||
1044 | } finally {
|
||
1045 | DisposeUtils.disposeQuietly(it); |
||
1046 | DisposeUtils.disposeQuietly(set); |
||
1047 | taskStatus.remove(); |
||
1048 | } |
||
1049 | } |
||
1050 | |||
1051 | public File getAuxFile(String extension) { |
||
1052 | File data_file = CSVStoreParameters.getFile(this.getCSVParameters()); |
||
1053 | if (data_file == null){ |
||
1054 | return null; |
||
1055 | } |
||
1056 | File index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + "." + extension); |
||
1057 | return index_file;
|
||
1058 | } |
||
1059 | |||
1060 | public SpatialIndex getSpatialIndex() {
|
||
1061 | return spatialIndex;
|
||
1062 | } |
||
1063 | |||
1064 | private void bboxFileSave(Envelope envelope) { |
||
1065 | File bboxfile = this.getAuxFile("bbox"); |
||
1066 | bboxFileSave(bboxfile, envelope); |
||
1067 | } |
||
1068 | |||
1069 | private void bboxFileSave(File bboxfile, Envelope envelope) { |
||
1070 | if( envelope == null ) { |
||
1071 | bboxfile.delete(); |
||
1072 | return;
|
||
1073 | } |
||
1074 | try {
|
||
1075 | FileUtils.write( |
||
1076 | bboxfile, |
||
1077 | envelope.getBox2D().convertToWKTQuietly(), |
||
1078 | StandardCharsets.UTF_8 |
||
1079 | ); |
||
1080 | } catch(Exception ex) { |
||
1081 | LOGGER.warn("Can't write bbox file '"+Objects.toString(bboxfile)+"'.",ex); |
||
1082 | } |
||
1083 | } |
||
1084 | |||
1085 | private Envelope bboxFileLoad() {
|
||
1086 | File bboxfile = this.getAuxFile("bbox"); |
||
1087 | return bboxFileLoad(bboxfile);
|
||
1088 | } |
||
1089 | |||
1090 | private Envelope bboxFileLoad(File bboxfile) { |
||
1091 | try {
|
||
1092 | if( bboxfile.exists() ) {
|
||
1093 | GeometryManager geomManager = GeometryLocator.getGeometryManager(); |
||
1094 | String wkt = FileUtils.readFileToString(bboxfile, StandardCharsets.UTF_8);
|
||
1095 | Geometry geom = geomManager.createFrom(wkt); |
||
1096 | if( geom!=null ) { |
||
1097 | return geom.getEnvelope();
|
||
1098 | } |
||
1099 | } |
||
1100 | } catch(Exception ex) { |
||
1101 | LOGGER.warn("Can't load bbox file",ex);
|
||
1102 | } |
||
1103 | return null; |
||
1104 | } |
||
1105 | |||
1106 | 47634 | fdiaz | public SimpleReader getSimpleReader(CSVStoreParameters parameters, Reader in) throws IOException { |
1107 | return CSVUtils.getSimpleReader(getCSVParameters(), in);
|
||
1108 | } |
||
1109 | |||
1110 | 40846 | jjdelcerro | } |