Revision 42775 trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreProvider.java
| CSVStoreProvider.java | ||
|---|---|---|
| 3 | 3 |
* |
| 4 | 4 |
* Copyright (C) 2007-2013 gvSIG Association. |
| 5 | 5 |
* |
| 6 |
* This program is free software; you can redistribute it and/or |
|
| 7 |
* modify it under the terms of the GNU General Public License
|
|
| 8 |
* as published by the Free Software Foundation; either version 3
|
|
| 9 |
* of the License, or (at your option) any later version.
|
|
| 6 |
* This program is free software; you can redistribute it and/or modify it under
|
|
| 7 |
* the terms of the GNU General Public License as published by the Free Software
|
|
| 8 |
* Foundation; either version 3 of the License, or (at your option) any later
|
|
| 9 |
* version. |
|
| 10 | 10 |
* |
| 11 |
* This program is distributed in the hope that it will be useful, |
|
| 12 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
| 13 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
| 14 |
* GNU General Public License for more details.
|
|
| 11 |
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
| 12 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
| 13 |
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
|
| 14 |
* details. |
|
| 15 | 15 |
* |
| 16 |
* You should have received a copy of the GNU General Public License |
|
| 17 |
* along with this program; if not, write to the Free Software |
|
| 18 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|
| 19 |
* MA 02110-1301, USA. |
|
| 16 |
* You should have received a copy of the GNU General Public License along with |
|
| 17 |
* this program; if not, write to the Free Software Foundation, Inc., 51 |
|
| 18 |
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. |
|
| 20 | 19 |
* |
| 21 |
* For any additional information, do not hesitate to contact us |
|
| 22 |
* at info AT gvsig.com, or visit our website www.gvsig.com.
|
|
| 20 |
* For any additional information, do not hesitate to contact us at info AT
|
|
| 21 |
* gvsig.com, or visit our website www.gvsig.com. |
|
| 23 | 22 |
*/ |
| 24 | 23 |
package org.gvsig.fmap.dal.store.csv; |
| 25 | 24 |
|
| ... | ... | |
| 36 | 35 |
import java.util.Locale; |
| 37 | 36 |
|
| 38 | 37 |
import org.apache.commons.io.FilenameUtils; |
| 38 |
import org.apache.commons.io.IOUtils; |
|
| 39 | 39 |
import org.apache.commons.lang3.StringUtils; |
| 40 | 40 |
import org.cresques.cts.IProjection; |
| 41 | 41 |
import org.gvsig.fmap.dal.DALLocator; |
| ... | ... | |
| 168 | 168 |
} |
| 169 | 169 |
|
| 170 | 170 |
public void open() throws OpenException {
|
| 171 |
if ( this.data != null ) {
|
|
| 171 |
if (this.data != null) {
|
|
| 172 | 172 |
return; |
| 173 | 173 |
} |
| 174 | 174 |
this.data = new ArrayList<FeatureProvider>(); |
| ... | ... | |
| 222 | 222 |
this.ftype = ftype; |
| 223 | 223 |
this.csvpreferences = csvpreferences; |
| 224 | 224 |
this.locale = CSVStoreParameters.getLocale(getCSVParameters()); |
| 225 |
if ( csvpreferences == null ) {
|
|
| 225 |
if (csvpreferences == null) {
|
|
| 226 | 226 |
this.csvpreferences = CsvPreference.STANDARD_PREFERENCE; |
| 227 | 227 |
} |
| 228 |
if ( ftype.getDefaultGeometryAttributeName() != null ) {
|
|
| 228 |
if (ftype.getDefaultGeometryAttributeName() != null) {
|
|
| 229 | 229 |
this.calculate_envelope = true; |
| 230 | 230 |
} |
| 231 | 231 |
this.descriptors = this.ftype.getAttributeDescriptors(); |
| ... | ... | |
| 242 | 242 |
} |
| 243 | 243 |
this.listWriter = new CsvListWriter(this.fwriter, this.csvpreferences); |
| 244 | 244 |
int n = 0; |
| 245 |
for ( int i = 0; i < descriptors.length; i++ ) {
|
|
| 245 |
for (int i = 0; i < descriptors.length; i++) {
|
|
| 246 | 246 |
FeatureAttributeDescriptor descriptor = descriptors[i]; |
| 247 |
if ( descriptor.getEvaluator() == null ) {
|
|
| 247 |
if (descriptor.getEvaluator() == null) {
|
|
| 248 | 248 |
n++; |
| 249 | 249 |
} |
| 250 | 250 |
} |
| ... | ... | |
| 252 | 252 |
String[] header = new String[n]; |
| 253 | 253 |
this.values = new String[n]; |
| 254 | 254 |
n = 0; |
| 255 |
for ( int i = 0; i < descriptors.length; i++ ) {
|
|
| 255 |
for (int i = 0; i < descriptors.length; i++) {
|
|
| 256 | 256 |
FeatureAttributeDescriptor descriptor = descriptors[i]; |
| 257 |
if ( descriptor.getEvaluator() == null ) {
|
|
| 257 |
if (descriptor.getEvaluator() == null) {
|
|
| 258 | 258 |
String name = descriptor.getName(); |
| 259 | 259 |
String typeName = descriptor.getDataTypeName(); |
| 260 |
if ( descriptor.getDataType().getType() == DataTypes.STRING ) {
|
|
| 260 |
if (descriptor.getDataType().getType() == DataTypes.STRING) {
|
|
| 261 | 261 |
header[n++] = name + "__" + typeName + "__" + descriptor.getSize(); |
| 262 | 262 |
} else {
|
| 263 | 263 |
header[n++] = name + "__" + typeName; |
| ... | ... | |
| 273 | 273 |
} |
| 274 | 274 |
|
| 275 | 275 |
public void add(FeatureProvider feature) {
|
| 276 |
if ( this.calculate_envelope ) {
|
|
| 276 |
if (this.calculate_envelope) {
|
|
| 277 | 277 |
Geometry geom = feature.getDefaultGeometry(); |
| 278 |
if ( geom != null ) {
|
|
| 279 |
if ( envelope == null ) {
|
|
| 278 |
if (geom != null) {
|
|
| 279 |
if (envelope == null) {
|
|
| 280 | 280 |
try {
|
| 281 | 281 |
envelope = (Envelope) geom.getEnvelope().clone(); |
| 282 | 282 |
} catch (CloneNotSupportedException e) {
|
| ... | ... | |
| 288 | 288 |
} |
| 289 | 289 |
} |
| 290 | 290 |
int n = 0; |
| 291 |
for ( int i = 0; i < descriptors.length; i++ ) {
|
|
| 291 |
for (int i = 0; i < descriptors.length; i++) {
|
|
| 292 | 292 |
FeatureAttributeDescriptor descriptor = descriptors[i]; |
| 293 |
if ( descriptor.getEvaluator() == null ) {
|
|
| 293 |
if (descriptor.getEvaluator() == null) {
|
|
| 294 | 294 |
Object value = feature.get(i); |
| 295 | 295 |
try {
|
| 296 | 296 |
n++; |
| 297 |
if ( this.convert != null && this.convert instanceof CoercionWithLocale ) {
|
|
| 297 |
if (this.convert != null && this.convert instanceof CoercionWithLocale) {
|
|
| 298 | 298 |
values[n] = (String) ((CoercionWithLocale) this.convert).coerce(value, this.locale); |
| 299 | 299 |
} else {
|
| 300 | 300 |
values[n] = (String) this.convert.coerce(value); |
| ... | ... | |
| 305 | 305 |
} catch (Exception ex) {
|
| 306 | 306 |
values[n] = ""; |
| 307 | 307 |
} |
| 308 |
if ( errorcounts++ <= 10 ) {
|
|
| 308 |
if (errorcounts++ <= 10) {
|
|
| 309 | 309 |
this.lasterror = e; |
| 310 | 310 |
logger.warn("Can't convert value of field " + i + " to string in CVS file '" + getFullFileName() + "'.", e);
|
| 311 |
if ( errorcounts == 10 ) {
|
|
| 311 |
if (errorcounts == 10) {
|
|
| 312 | 312 |
logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");
|
| 313 | 313 |
} |
| 314 | 314 |
} |
| ... | ... | |
| 318 | 318 |
try {
|
| 319 | 319 |
this.listWriter.writeHeader(values); |
| 320 | 320 |
} catch (IOException e) {
|
| 321 |
if ( errorcounts++ <= 10 ) {
|
|
| 321 |
if (errorcounts++ <= 10) {
|
|
| 322 | 322 |
this.lasterror = e; |
| 323 | 323 |
logger.warn("Can't write values to CVS file '" + getFullFileName() + "'.", e);
|
| 324 |
if ( errorcounts == 10 ) {
|
|
| 324 |
if (errorcounts == 10) {
|
|
| 325 | 325 |
logger.warn("Too many error writing CVS file '" + getFullFileName() + "', don't output more.");
|
| 326 | 326 |
} |
| 327 | 327 |
} |
| ... | ... | |
| 330 | 330 |
} |
| 331 | 331 |
|
| 332 | 332 |
public void end() throws PerformEditingException {
|
| 333 |
if ( this.errorcounts > 0 ) {
|
|
| 333 |
if (this.errorcounts > 0) {
|
|
| 334 | 334 |
throw new PerformEditingException(this.file.getAbsolutePath(), lasterror); |
| 335 | 335 |
} |
| 336 |
if ( listWriter != null ) {
|
|
| 336 |
if (listWriter != null) {
|
|
| 337 | 337 |
try {
|
| 338 | 338 |
listWriter.close(); |
| 339 | 339 |
} catch (Exception ex) {
|
| ... | ... | |
| 341 | 341 |
} |
| 342 | 342 |
listWriter = null; |
| 343 | 343 |
} |
| 344 |
if ( fwriter != null ) {
|
|
| 344 |
if (fwriter != null) {
|
|
| 345 | 345 |
try {
|
| 346 | 346 |
fwriter.close(); |
| 347 | 347 |
} catch (Exception ex) {
|
| ... | ... | |
| 378 | 378 |
it = features.fastIterator(); |
| 379 | 379 |
taskStatus.setRangeOfValues(0, 0); |
| 380 | 380 |
long counter = 0; |
| 381 |
while ( it.hasNext() ) {
|
|
| 381 |
while (it.hasNext()) {
|
|
| 382 | 382 |
taskStatus.setCurValue(counter++); |
| 383 | 383 |
FeatureProvider feature = getStoreServices().getFeatureProviderFromFeature( |
| 384 | 384 |
(org.gvsig.fmap.dal.feature.Feature) it.next()); |
| 385 | 385 |
writer.add(feature); |
| 386 |
if ( feature.getOID() == null ) {
|
|
| 386 |
if (feature.getOID() == null) {
|
|
| 387 | 387 |
logger.warn("feature without OID");
|
| 388 | 388 |
feature.setOID(createNewOID()); |
| 389 | 389 |
} |
| 390 | 390 |
newdata.add(feature); |
| 391 | 391 |
} |
| 392 | 392 |
data = newdata; |
| 393 |
if ( writer.getEnvelope() != null ) {
|
|
| 393 |
if (writer.getEnvelope() != null) {
|
|
| 394 | 394 |
envelope = writer.getEnvelope().getGeometry().getEnvelope(); |
| 395 | 395 |
} |
| 396 | 396 |
resource.notifyChanges(); |
| 397 | 397 |
writer.end(); |
| 398 | 398 |
} finally {
|
| 399 |
if ( it != null ) {
|
|
| 399 |
if (it != null) {
|
|
| 400 | 400 |
it.dispose(); |
| 401 | 401 |
} |
| 402 |
if ( features != null ) {
|
|
| 402 |
if (features != null) {
|
|
| 403 | 403 |
features.dispose(); |
| 404 | 404 |
} |
| 405 | 405 |
} |
| ... | ... | |
| 467 | 467 |
|
| 468 | 468 |
public Envelope getEnvelope() throws DataException {
|
| 469 | 469 |
this.open(); |
| 470 |
if ( this.envelope != null ) {
|
|
| 470 |
if (this.envelope != null) {
|
|
| 471 | 471 |
return this.envelope; |
| 472 | 472 |
} |
| 473 |
if ( !this.need_calculate_envelope ) {
|
|
| 473 |
if (!this.need_calculate_envelope) {
|
|
| 474 | 474 |
return null; |
| 475 | 475 |
} |
| 476 | 476 |
FeatureStore fs = this.getFeatureStore(); |
| ... | ... | |
| 483 | 483 |
public void visit(Object obj) throws VisitCanceledException, BaseException {
|
| 484 | 484 |
Feature f = (Feature) obj; |
| 485 | 485 |
Geometry geom = f.getDefaultGeometry(); |
| 486 |
if ( geom != null ) {
|
|
| 486 |
if (geom != null) {
|
|
| 487 | 487 |
envelope.add(geom.getEnvelope()); |
| 488 | 488 |
} |
| 489 | 489 |
} |
| ... | ... | |
| 498 | 498 |
} |
| 499 | 499 |
|
| 500 | 500 |
public Object getDynValue(String name) throws DynFieldNotFoundException {
|
| 501 |
if ( DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name) ) {
|
|
| 501 |
if (DataStore.METADATA_ENVELOPE.equalsIgnoreCase(name)) {
|
|
| 502 | 502 |
try {
|
| 503 | 503 |
return this.getEnvelope(); |
| 504 | 504 |
} catch (DataException e) {
|
| 505 | 505 |
return null; |
| 506 | 506 |
} |
| 507 | 507 |
} else {
|
| 508 |
if ( DataStore.METADATA_CRS.equalsIgnoreCase(name) ) {
|
|
| 508 |
if (DataStore.METADATA_CRS.equalsIgnoreCase(name)) {
|
|
| 509 | 509 |
IProjection pro = CSVStoreParameters.getCRS(this.getCSVParameters()); |
| 510 |
if ( pro != null ) {
|
|
| 510 |
if (pro != null) {
|
|
| 511 | 511 |
return pro; |
| 512 | 512 |
} |
| 513 | 513 |
} |
| ... | ... | |
| 539 | 539 |
} |
| 540 | 540 |
|
| 541 | 541 |
private boolean isEmpty(String s) {
|
| 542 |
if ( s == null ) {
|
|
| 542 |
if (s == null) {
|
|
| 543 | 543 |
return true; |
| 544 | 544 |
} |
| 545 | 545 |
return s.trim().length() == 0; |
| ... | ... | |
| 563 | 563 |
return dataTypesManager.getType(typename); |
| 564 | 564 |
} |
| 565 | 565 |
|
| 566 |
public void clear() {
|
|
| 567 |
name = null; |
|
| 568 |
type = DataTypes.STRING; |
|
| 569 |
size = 0; |
|
| 570 |
allowNulls = true; |
|
| 571 |
geometryType = Geometry.TYPES.GEOMETRY; |
|
| 572 |
} |
|
| 573 |
|
|
| 574 |
public void copyFrom(FieldTypeParser other) {
|
|
| 575 |
name = other.name; |
|
| 576 |
type = other.type; |
|
| 577 |
size = other.size; |
|
| 578 |
allowNulls = other.allowNulls; |
|
| 579 |
geometryType = other.geometryType; |
|
| 580 |
} |
|
| 581 |
|
|
| 566 | 582 |
// El formato seria: |
| 567 | 583 |
// name[:typename[:size[:notnull|null]]] |
| 568 | 584 |
// name[:GEOMETRY[:geometry_type[:notnull|null]]] |
| ... | ... | |
| 571 | 587 |
public boolean parse(String value) {
|
| 572 | 588 |
String typename = null; |
| 573 | 589 |
String[] ss = null; |
| 574 |
if ( value.contains(":") ) {
|
|
| 590 |
if (value.contains(":")) {
|
|
| 575 | 591 |
ss = value.split(":");
|
| 576 |
} else if ( value.contains("__") ) {
|
|
| 592 |
} else if (value.contains("__")) {
|
|
| 577 | 593 |
ss = value.split("__");
|
| 578 | 594 |
} |
| 579 |
if ( ss == null ) {
|
|
| 595 |
if (ss == null) {
|
|
| 580 | 596 |
this.name = value; |
| 581 | 597 |
return true; |
| 582 | 598 |
} |
| 583 | 599 |
switch (ss.length) {
|
| 584 |
case 4: |
|
| 585 |
if ( !StringUtils.isBlank(ss[3]) ) {
|
|
| 586 |
if ( "notnull".equalsIgnoreCase(ss[3].trim()) ) {
|
|
| 587 |
this.allowNulls = false; |
|
| 588 |
} else {
|
|
| 589 |
this.allowNulls = true; |
|
| 600 |
case 4: |
|
| 601 |
if (!StringUtils.isBlank(ss[3])) {
|
|
| 602 |
if ("notnull".equalsIgnoreCase(ss[3].trim())) {
|
|
| 603 |
this.allowNulls = false; |
|
| 604 |
} else {
|
|
| 605 |
this.allowNulls = true; |
|
| 606 |
} |
|
| 590 | 607 |
} |
| 591 |
}
|
|
| 592 |
case 3:
|
|
| 593 |
if ( !StringUtils.isBlank(ss[1]) ) {
|
|
| 594 |
this.typename = ss[1].trim();
|
|
| 595 |
this.type = this.getType(this.typename);
|
|
| 596 |
if ( this.type == DataTypes.INVALID ) {
|
|
| 597 |
this.type = DataTypes.STRING;
|
|
| 598 |
logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");
|
|
| 608 |
case 3:
|
|
| 609 |
if (!StringUtils.isBlank(ss[1])) {
|
|
| 610 |
this.typename = ss[1].trim();
|
|
| 611 |
this.type = this.getType(this.typename);
|
|
| 612 |
if (this.type == DataTypes.INVALID) {
|
|
| 613 |
this.type = DataTypes.STRING;
|
|
| 614 |
logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");
|
|
| 615 |
}
|
|
| 599 | 616 |
} |
| 600 |
} |
|
| 601 |
if ( !StringUtils.isBlank(ss[2]) ) {
|
|
| 602 |
if( this.type == DataTypes.GEOMETRY ) {
|
|
| 603 |
String s = ss[2].trim(); |
|
| 604 |
if( s.equalsIgnoreCase("line") || s.equalsIgnoreCase("linestring") || s.equalsIgnoreCase("curve") ) {
|
|
| 605 |
this.geometryType = Geometry.TYPES.CURVE; |
|
| 606 |
} else if( s.equalsIgnoreCase("multiline") || s.equalsIgnoreCase("multilinestring") || s.equalsIgnoreCase("multicurve") ) {
|
|
| 607 |
this.geometryType = Geometry.TYPES.MULTICURVE; |
|
| 608 |
} else if( s.equalsIgnoreCase("point") ) {
|
|
| 609 |
this.geometryType = Geometry.TYPES.POINT; |
|
| 610 |
} else if( s.equalsIgnoreCase("multipoint") ) {
|
|
| 611 |
this.geometryType = Geometry.TYPES.MULTIPOINT; |
|
| 612 |
} else if( s.equalsIgnoreCase("polygon") || s.equalsIgnoreCase("surface") ) {
|
|
| 613 |
this.geometryType = Geometry.TYPES.POLYGON; |
|
| 614 |
} else if( s.equalsIgnoreCase("multipolygon") || s.equalsIgnoreCase("multisurface") ) {
|
|
| 615 |
this.geometryType = Geometry.TYPES.MULTISURFACE; |
|
| 617 |
if (!StringUtils.isBlank(ss[2])) {
|
|
| 618 |
if (this.type == DataTypes.GEOMETRY) {
|
|
| 619 |
String s = ss[2].trim(); |
|
| 620 |
if (s.equalsIgnoreCase("line") || s.equalsIgnoreCase("linestring") || s.equalsIgnoreCase("curve")) {
|
|
| 621 |
this.geometryType = Geometry.TYPES.CURVE; |
|
| 622 |
} else if (s.equalsIgnoreCase("multiline") || s.equalsIgnoreCase("multilinestring") || s.equalsIgnoreCase("multicurve")) {
|
|
| 623 |
this.geometryType = Geometry.TYPES.MULTICURVE; |
|
| 624 |
} else if (s.equalsIgnoreCase("point")) {
|
|
| 625 |
this.geometryType = Geometry.TYPES.POINT; |
|
| 626 |
} else if (s.equalsIgnoreCase("multipoint")) {
|
|
| 627 |
this.geometryType = Geometry.TYPES.MULTIPOINT; |
|
| 628 |
} else if (s.equalsIgnoreCase("polygon") || s.equalsIgnoreCase("surface")) {
|
|
| 629 |
this.geometryType = Geometry.TYPES.POLYGON; |
|
| 630 |
} else if (s.equalsIgnoreCase("multipolygon") || s.equalsIgnoreCase("multisurface")) {
|
|
| 631 |
this.geometryType = Geometry.TYPES.MULTISURFACE; |
|
| 632 |
} |
|
| 633 |
this.size = 1; |
|
| 634 |
} else {
|
|
| 635 |
try {
|
|
| 636 |
this.size = Integer.parseInt(ss[2]); |
|
| 637 |
} catch (Exception ex) {
|
|
| 638 |
logger.warn("Ignore incorrect field size for field " + value + " in CSV header of '" + getFullFileName() + "'.", ex);
|
|
| 639 |
} |
|
| 616 | 640 |
} |
| 617 |
this.size = 1; |
|
| 618 |
} else {
|
|
| 619 |
try {
|
|
| 620 |
this.size = Integer.parseInt(ss[2]); |
|
| 621 |
} catch (Exception ex) {
|
|
| 622 |
logger.warn("Ignore incorrect field size for field " + value + " in CSV header of '" + getFullFileName() + "'.", ex);
|
|
| 641 |
} |
|
| 642 |
this.name = ss[0].trim(); |
|
| 643 |
break; |
|
| 644 |
case 2: |
|
| 645 |
if (!StringUtils.isBlank(ss[1])) {
|
|
| 646 |
this.typename = ss[1].trim(); |
|
| 647 |
this.type = this.getType(this.typename); |
|
| 648 |
if (this.type == DataTypes.INVALID) {
|
|
| 649 |
this.type = DataTypes.STRING; |
|
| 650 |
logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");
|
|
| 623 | 651 |
} |
| 624 | 652 |
} |
| 625 |
} |
|
| 626 |
this.name = ss[0].trim(); |
|
| 627 |
break; |
|
| 628 |
case 2: |
|
| 629 |
if ( !StringUtils.isBlank(ss[1]) ) {
|
|
| 630 |
this.typename = ss[1].trim(); |
|
| 631 |
this.type = this.getType(this.typename); |
|
| 632 |
if ( this.type == DataTypes.INVALID ) {
|
|
| 633 |
this.type = DataTypes.STRING; |
|
| 634 |
logger.info("Type '" + typename + "' not valid for attribute '" + value + "' in CSV file '" + getFullFileName() + "'.");
|
|
| 635 |
} |
|
| 636 |
} |
|
| 637 |
case 1: |
|
| 638 |
this.name = ss[0].trim(); |
|
| 639 |
break; |
|
| 653 |
case 1: |
|
| 654 |
this.name = ss[0].trim(); |
|
| 655 |
break; |
|
| 640 | 656 |
} |
| 641 | 657 |
|
| 642 |
if ( this.type != DataTypes.STRING ) {
|
|
| 658 |
if (this.type != DataTypes.STRING) {
|
|
| 643 | 659 |
this.size = 0; |
| 644 | 660 |
} |
| 645 | 661 |
return true; |
| ... | ... | |
| 655 | 671 |
// |
| 656 | 672 |
// Calculamos cuales pueden ser los tipos de datos |
| 657 | 673 |
// |
| 658 |
for ( int i = 0; i < fieldTypes.length; i++ ) {
|
|
| 674 |
for (int i = 0; i < fieldTypes.length; i++) {
|
|
| 659 | 675 |
fieldTypes[i] = new FieldTypeParser(); |
| 660 | 676 |
} |
| 661 | 677 |
|
| 662 | 678 |
// Asuminos los tipos pasados por parametro, que se supone |
| 663 | 679 |
// son los detectados automaticamente. |
| 664 |
if ( automaticTypes != null ) {
|
|
| 665 |
for ( int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++ ) {
|
|
| 680 |
if (automaticTypes != null) {
|
|
| 681 |
for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) {
|
|
| 666 | 682 |
fieldTypes[i].type = automaticTypes[i]; |
| 667 | 683 |
} |
| 668 | 684 |
} |
| 669 | 685 |
// Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo |
| 670 | 686 |
// los tipos anteriores en caso de definirse en la cabezara. |
| 671 |
for ( int i = 0; i < fieldTypes.length; i++ ) {
|
|
| 672 |
if ( !fieldTypes[i].parse(headers[i]) ) {
|
|
| 687 |
for (int i = 0; i < fieldTypes.length; i++) {
|
|
| 688 |
if (!fieldTypes[i].parse(headers[i])) {
|
|
| 673 | 689 |
continue; |
| 674 | 690 |
} |
| 675 | 691 |
|
| ... | ... | |
| 680 | 696 |
String param_types_def = CSVStoreParameters.getRawFieldTypes(this.getParameters()); |
| 681 | 697 |
if (StringUtils.isNotBlank(param_types_def)) {
|
| 682 | 698 |
String sep = CSVStoreParameters.getDelimiter(param_types_def); |
| 683 |
if ( StringUtils.isNotBlank(sep) ) {
|
|
| 699 |
if (StringUtils.isNotBlank(sep)) {
|
|
| 684 | 700 |
String[] param_types = param_types_def.split(sep); |
| 685 |
for (int i = 0; i < fieldTypes.length && i < param_types.length; i++) {
|
|
| 686 |
if (!fieldTypes[i].parse(param_types[i])) {
|
|
| 687 |
continue; |
|
| 701 |
FieldTypeParser parser = new FieldTypeParser(); |
|
| 702 |
for (String param_type : param_types) {
|
|
| 703 |
parser.clear(); |
|
| 704 |
parser.parse(param_type); |
|
| 705 |
for (FieldTypeParser fieldType : fieldTypes) {
|
|
| 706 |
if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) {
|
|
| 707 |
fieldType.copyFrom(parser); |
|
| 708 |
break; |
|
| 709 |
} |
|
| 688 | 710 |
} |
| 689 |
|
|
| 690 | 711 |
} |
| 691 | 712 |
} |
| 692 | 713 |
} |
| 693 | 714 |
// |
| 694 | 715 |
// Una vez ya sabemos los tipos de datos rellenamos el feature-type |
| 695 | 716 |
// |
| 696 |
for ( int i = 0; i < fieldTypes.length; i++ ) {
|
|
| 717 |
for (int i = 0; i < fieldTypes.length; i++) {
|
|
| 697 | 718 |
EditableFeatureAttributeDescriptor fad = fType.add( |
| 698 | 719 |
fieldTypes[i].name, |
| 699 | 720 |
fieldTypes[i].type |
| 700 | 721 |
); |
| 701 | 722 |
fad.setSize(fieldTypes[i].size); |
| 702 | 723 |
fad.setAllowNull(fieldTypes[i].allowNulls); |
| 703 |
if ( fieldTypes[i].type == DataTypes.GEOMETRY ) {
|
|
| 704 |
if( fType.getDefaultGeometryAttributeName() == null ) {
|
|
| 724 |
if (fieldTypes[i].type == DataTypes.GEOMETRY) {
|
|
| 725 |
if (fType.getDefaultGeometryAttributeName() == null) {
|
|
| 705 | 726 |
fType.setDefaultGeometryAttributeName(fieldTypes[i].name); |
| 706 | 727 |
} |
| 707 | 728 |
fad.setGeometryType(fieldTypes[i].geometryType); |
| 708 | 729 |
} |
| 709 | 730 |
} |
| 710 | 731 |
String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(this.getParameters()); |
| 711 |
if ( pointDimensionNames != null ) {
|
|
| 732 |
if (pointDimensionNames != null) {
|
|
| 712 | 733 |
PointAttributeEmulator emulator = new PointAttributeEmulator(pointDimensionNames); |
| 713 |
EditableFeatureAttributeDescriptor attr = fType.add("GEOM", DataTypes.GEOMETRY, emulator);
|
|
| 734 |
EditableFeatureAttributeDescriptor attr = fType.add( |
|
| 735 |
CSVStoreParameters.getPointColumnName(this.getParameters()), |
|
| 736 |
DataTypes.GEOMETRY, emulator |
|
| 737 |
); |
|
| 714 | 738 |
GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
| 715 | 739 |
GeometryType gt; |
| 716 | 740 |
try {
|
| ... | ... | |
| 740 | 764 |
private int errorcount = 0; |
| 741 | 765 |
|
| 742 | 766 |
public PointAttributeEmulator(String[] pointDimensionNames) {
|
| 743 |
if ( pointDimensionNames.length > 2 ) {
|
|
| 767 |
if (pointDimensionNames.length > 2) {
|
|
| 744 | 768 |
this.fieldNames = new String[3]; |
| 745 | 769 |
this.fieldNames[ZNAME] = pointDimensionNames[2]; |
| 746 | 770 |
} else {
|
| ... | ... | |
| 759 | 783 |
try {
|
| 760 | 784 |
Object valueX = feature.get(this.fieldNames[XNAME]); |
| 761 | 785 |
valueX = toDouble.coerce(valueX); |
| 762 |
if ( valueX == null ) {
|
|
| 786 |
if (valueX == null) {
|
|
| 763 | 787 |
return null; |
| 764 | 788 |
} |
| 765 | 789 |
Object valueY = feature.get(this.fieldNames[YNAME]); |
| 766 | 790 |
valueY = toDouble.coerce(valueY); |
| 767 |
if ( valueY == null ) {
|
|
| 791 |
if (valueY == null) {
|
|
| 768 | 792 |
return null; |
| 769 | 793 |
} |
| 770 | 794 |
Object valueZ = null; |
| 771 |
if ( this.fieldNames.length > 2 ) {
|
|
| 795 |
if (this.fieldNames.length > 2) {
|
|
| 772 | 796 |
valueZ = toDouble.coerce(feature.get(this.fieldNames[ZNAME])); |
| 773 |
if ( valueZ == null ) {
|
|
| 797 |
if (valueZ == null) {
|
|
| 774 | 798 |
return null; |
| 775 | 799 |
} |
| 776 | 800 |
} |
| ... | ... | |
| 778 | 802 |
double x = ((Double) valueX).doubleValue(); |
| 779 | 803 |
double y = ((Double) valueY).doubleValue(); |
| 780 | 804 |
Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D); |
| 781 |
if ( this.fieldNames.length > 2 ) {
|
|
| 805 |
if (this.fieldNames.length > 2) {
|
|
| 782 | 806 |
double z = ((Double) valueZ).doubleValue(); |
| 783 | 807 |
point.setCoordinateAt(2, z); |
| 784 | 808 |
} |
| 785 | 809 |
return point; |
| 786 | 810 |
} catch (Exception ex) {
|
| 787 |
if ( ++errorcount < 5 ) {
|
|
| 811 |
if (++errorcount < 5) {
|
|
| 788 | 812 |
logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"
|
| 789 | 813 |
+ this.fieldNames[XNAME] + "', YNAME='" + this.fieldNames[XNAME] + "' feature=" + feature.toString(), ex); |
| 790 | 814 |
} |
| ... | ... | |
| 793 | 817 |
} |
| 794 | 818 |
|
| 795 | 819 |
public void set(EditableFeature feature, Object value) {
|
| 796 |
if ( value == null ) {
|
|
| 820 |
if (value == null) {
|
|
| 797 | 821 |
return; |
| 798 | 822 |
} |
| 799 | 823 |
Point point = null; |
| 800 |
if ( value instanceof MultiPoint ) {
|
|
| 824 |
if (value instanceof MultiPoint) {
|
|
| 801 | 825 |
point = (Point) ((MultiPoint) value).getPrimitiveAt(0); |
| 802 | 826 |
} else {
|
| 803 | 827 |
point = (Point) value; |
| 804 | 828 |
} |
| 805 | 829 |
feature.set(this.fieldNames[XNAME], point.getX()); |
| 806 | 830 |
feature.set(this.fieldNames[YNAME], point.getY()); |
| 807 |
if ( this.fieldNames.length > 2 ) {
|
|
| 831 |
if (this.fieldNames.length > 2) {
|
|
| 808 | 832 |
feature.set(this.fieldNames[ZNAME], point.getCoordinateAt(2)); |
| 809 | 833 |
} |
| 810 | 834 |
} |
| ... | ... | |
| 833 | 857 |
ToPointEvaluaror(String[] pointDimensionNames) {
|
| 834 | 858 |
this.xname = pointDimensionNames[0]; |
| 835 | 859 |
this.yname = pointDimensionNames[1]; |
| 836 |
if ( pointDimensionNames.length > 2 ) {
|
|
| 860 |
if (pointDimensionNames.length > 2) {
|
|
| 837 | 861 |
this.zname = pointDimensionNames[2]; |
| 838 | 862 |
} |
| 839 | 863 |
this.geommgr = GeometryLocator.getGeometryManager(); |
| ... | ... | |
| 845 | 869 |
double x = ((Double) toDouble.coerce(data.getDataValue(xname))).doubleValue(); |
| 846 | 870 |
double y = ((Double) toDouble.coerce(data.getDataValue(yname))).doubleValue(); |
| 847 | 871 |
Point point = geommgr.createPoint(x, y, Geometry.SUBTYPES.GEOM3D); |
| 848 |
if ( zname != null ) {
|
|
| 872 |
if (zname != null) {
|
|
| 849 | 873 |
double z = ((Double) toDouble.coerce(data.getDataValue(zname))).doubleValue(); |
| 850 | 874 |
point.setCoordinateAt(2, z); |
| 851 | 875 |
} |
| 852 | 876 |
return point; |
| 853 | 877 |
} catch (Exception ex) {
|
| 854 |
if ( ++errorcount < 5 ) {
|
|
| 878 |
if (++errorcount < 5) {
|
|
| 855 | 879 |
logger.warn("[" + errorcount + "] Can't create point in CSV provider. XNAME='"
|
| 856 | 880 |
+ xname + "', YNAME='" + yname + "', ZNAME='" + zname + "', data=" + data.toString()); |
| 857 | 881 |
} |
| ... | ... | |
| 867 | 891 |
|
| 868 | 892 |
private SimpleReader getSimpleReader(FileReader in) {
|
| 869 | 893 |
SimpleReader reader; |
| 870 |
if ( CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null ) {
|
|
| 894 |
if (CSVStoreParameters.getRawFieldsDefinition(getCSVParameters()) != null) {
|
|
| 871 | 895 |
reader = new FixedLenReader(in, getCSVParameters()); |
| 872 | 896 |
} else {
|
| 873 | 897 |
reader = new CSVReader(in, getCSVParameters()); |
| ... | ... | |
| 875 | 899 |
return reader; |
| 876 | 900 |
} |
| 877 | 901 |
|
| 878 |
private void loadFeatures() throws IOException, DataException, |
|
| 879 |
CoercionException, CloneNotSupportedException {
|
|
| 902 |
private String getFixedHeader(int column) {
|
|
| 903 |
char[] header = new char[3]; |
|
| 904 |
|
|
| 905 |
String s = String.format("%03d", column);
|
|
| 906 |
header[0] = (char) (s.charAt(0) + 17); |
|
| 907 |
header[1] = (char) (s.charAt(1) + 17); |
|
| 908 |
header[2] = (char) (s.charAt(2) + 17); |
|
| 909 |
return String.valueOf(header); |
|
| 910 |
} |
|
| 911 |
|
|
| 912 |
private String[] getFixedHeaders(int count) {
|
|
| 913 |
String[] headers = new String[count]; |
|
| 914 |
for (int i = 0; i < headers.length; i++) {
|
|
| 915 |
headers[i] = getFixedHeader(i); |
|
| 916 |
} |
|
| 917 |
return headers; |
|
| 918 |
} |
|
| 919 |
|
|
| 920 |
private void loadFeatures() {
|
|
| 880 | 921 |
FileReader in = null; |
| 881 | 922 |
SimpleReader reader = null; |
| 882 | 923 |
try {
|
| ... | ... | |
| 890 | 931 |
reader = getSimpleReader(in); |
| 891 | 932 |
|
| 892 | 933 |
headers = CSVStoreParameters.getHeaders(getCSVParameters()); |
| 893 |
if ( headers == null ) {
|
|
| 894 |
headers = reader.getHeader(); |
|
| 895 |
if ( headers == null ) {
|
|
| 896 |
String msg = "Can't retrieve header from csv file '" |
|
| 897 |
+ this.getCSVParameters().getFile() |
|
| 898 |
.getAbsolutePath() |
|
| 899 |
+ "' and not specified in the parameters."; |
|
| 900 |
logger.warn(msg); |
|
| 901 |
throw new RuntimeException(msg); |
|
| 934 |
if (headers == null) {
|
|
| 935 |
if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
|
| 936 |
headers = reader.getHeader(); |
|
| 937 |
if (headers == null) {
|
|
| 938 |
if (CSVStoreParameters.getIgnoreErrors(getCSVParameters())) {
|
|
| 939 |
headers = getFixedHeaders(reader.getColumnsCount()); |
|
| 940 |
} else {
|
|
| 941 |
String msg = "Can't retrieve header from csv file '" |
|
| 942 |
+ this.getCSVParameters().getFile() |
|
| 943 |
.getAbsolutePath() |
|
| 944 |
+ "' and not specified in the parameters."; |
|
| 945 |
logger.warn(msg); |
|
| 946 |
throw new RuntimeException(msg); |
|
| 947 |
} |
|
| 948 |
} |
|
| 949 |
} else {
|
|
| 950 |
headers = getFixedHeaders(reader.getColumnsCount()); |
|
| 902 | 951 |
} |
| 952 |
} else {
|
|
| 953 |
if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
|
| 954 |
reader.getHeader(); // Skip and ignore the header of file |
|
| 955 |
} |
|
| 903 | 956 |
} |
| 904 | 957 |
|
| 958 |
int[] detectedTypes = automaticDetectionOfTypes(headers); |
|
| 959 |
if( detectedTypes!=null && detectedTypes.length>headers.length ) {
|
|
| 960 |
// Se han detectado mas columnas que las que hay en la cabezera, |
|
| 961 |
// a?adimos mas columnas a la cabezera. |
|
| 962 |
String[] headers2 = new String[detectedTypes.length]; |
|
| 963 |
for( int i=0; i<headers2.length; i++ ) {
|
|
| 964 |
if( i<headers.length ) {
|
|
| 965 |
headers2[i] = headers[i]; |
|
| 966 |
} else {
|
|
| 967 |
headers2[i] = getFixedHeader(i); |
|
| 968 |
} |
|
| 969 |
} |
|
| 970 |
headers = headers2; |
|
| 971 |
} |
|
| 905 | 972 |
// Initialize the feature types |
| 906 |
EditableFeatureType edftype = this.getFeatureType(headers, automaticDetectionOfTypes());
|
|
| 973 |
EditableFeatureType edftype = this.getFeatureType(headers, detectedTypes);
|
|
| 907 | 974 |
FeatureType ftype = edftype.getNotEditableCopy(); |
| 908 |
List<FeatureType> ftypes = new ArrayList<FeatureType>();
|
|
| 975 |
List<FeatureType> ftypes = new ArrayList<>(); |
|
| 909 | 976 |
ftypes.add(ftype); |
| 910 | 977 |
store.setFeatureTypes(ftypes, ftype); |
| 911 | 978 |
|
| 912 | 979 |
Coercion coercion[] = new Coercion[ftype.size()]; |
| 913 | 980 |
int sizes[] = new int[ftype.size()]; |
| 914 |
for ( int i = 0; i < ftype.size(); i++ ) {
|
|
| 981 |
for (int i = 0; i < ftype.size(); i++) {
|
|
| 915 | 982 |
sizes[i] = -1; |
| 916 | 983 |
FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i); |
| 917 | 984 |
coercion[i] = ad.getDataType().getCoercion(); |
| 918 |
if ( ad.getDataType().getType() == DataTypes.STRING ) {
|
|
| 919 |
if ( ad.getSize() == 0 ) {
|
|
| 985 |
if (ad.getDataType().getType() == DataTypes.STRING) {
|
|
| 986 |
if (ad.getSize() == 0) {
|
|
| 920 | 987 |
// Es un string y no tiene un size asignado. |
| 921 | 988 |
// Lo ponemos a cero para calcularlo. |
| 922 | 989 |
sizes[i] = 0; |
| 923 | 990 |
} |
| 924 | 991 |
} |
| 925 | 992 |
} |
| 926 |
if ( ftype.getDefaultGeometryAttributeName() != null ) {
|
|
| 993 |
if (ftype.getDefaultGeometryAttributeName() != null) {
|
|
| 927 | 994 |
this.need_calculate_envelope = true; |
| 928 | 995 |
} |
| 929 | 996 |
|
| ... | ... | |
| 936 | 1003 |
List<String> row = reader.read(); |
| 937 | 1004 |
|
| 938 | 1005 |
int skipLines = CSVStoreParameters.getSkipLines(getCSVParameters()); |
| 939 |
if ( skipLines > 0 ) {
|
|
| 1006 |
if (skipLines > 0) {
|
|
| 940 | 1007 |
row = reader.skip(skipLines); |
| 941 | 1008 |
} |
| 942 |
|
|
| 943 |
while ( row != null ) {
|
|
| 1009 |
int limit = CSVStoreParameters.getLimit(getCSVParameters()); |
|
| 1010 |
while (row != null) {
|
|
| 944 | 1011 |
taskStatus.setCurValue(++count); |
| 945 | 1012 |
FeatureProvider feature = this.createFeatureProvider(ftype); |
| 946 |
for ( int i = 0; i < row.size(); i++ ) {
|
|
| 1013 |
for (int i = 0; i < row.size(); i++) {
|
|
| 947 | 1014 |
Object rawvalue = row.get(i); |
| 948 | 1015 |
try {
|
| 949 | 1016 |
Object value = null; |
| 950 |
if ( locale != null && coercion[i] instanceof CoercionWithLocale ) {
|
|
| 1017 |
if (locale != null && coercion[i] instanceof CoercionWithLocale) {
|
|
| 951 | 1018 |
value = ((CoercionWithLocale) (coercion[i])).coerce(rawvalue, locale); |
| 952 | 1019 |
} else {
|
| 953 | 1020 |
value = coercion[i].coerce(rawvalue); |
| 954 | 1021 |
} |
| 955 | 1022 |
feature.set(i, value); |
| 956 |
if ( sizes[i] >= 0 && value != null ) {
|
|
| 1023 |
if (sizes[i] >= 0 && value instanceof String ) {
|
|
| 957 | 1024 |
int x = ((String) value).length(); |
| 958 |
if ( sizes[i] < x ) {
|
|
| 1025 |
if (sizes[i] < x) {
|
|
| 959 | 1026 |
sizes[i] = x; |
| 960 | 1027 |
} |
| 961 | 1028 |
} |
| 962 |
} catch (RuntimeException ex) {
|
|
| 963 |
if ( !ignore_errors ) {
|
|
| 1029 |
} catch (Exception ex) {
|
|
| 1030 |
if (!ignore_errors) {
|
|
| 964 | 1031 |
throw ex; |
| 965 | 1032 |
} |
| 966 |
if ( count_errors++ < 10 ) {
|
|
| 1033 |
if (count_errors++ < 10) {
|
|
| 967 | 1034 |
logger.warn("Can't load value of attribute " + i + " in row " + count + ".", ex);
|
| 968 | 1035 |
} |
| 969 |
if ( count_errors == 10 ) {
|
|
| 1036 |
if (count_errors == 10) {
|
|
| 970 | 1037 |
logger.info("Too many errors, suppress messages.");
|
| 971 | 1038 |
} |
| 972 | 1039 |
} |
| 973 | 1040 |
} |
| 974 | 1041 |
this.addFeatureProvider(feature); |
| 1042 |
if( limit>0 ) {
|
|
| 1043 |
if( limit < this.data.size() ) {
|
|
| 1044 |
break; |
|
| 1045 |
} |
|
| 1046 |
} |
|
| 975 | 1047 |
row = reader.read(); |
| 976 | 1048 |
} |
| 977 |
for ( int i = 0; i < ftype.size(); i++ ) {
|
|
| 978 |
if ( sizes[i] > 0 ) {
|
|
| 1049 |
for (int i = 0; i < ftype.size(); i++) {
|
|
| 1050 |
if (sizes[i] > 0) {
|
|
| 979 | 1051 |
EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor) edftype.getAttributeDescriptor(i)); |
| 980 | 1052 |
efad.setSize(sizes[i]); |
| 981 | 1053 |
} |
| ... | ... | |
| 983 | 1055 |
// Volvemos a asignar al store el featuretype, ya que puede |
| 984 | 1056 |
// haber cambiado. |
| 985 | 1057 |
ftype = edftype.getNotEditableCopy(); |
| 986 |
ftypes = new ArrayList<FeatureType>();
|
|
| 1058 |
ftypes = new ArrayList<>(); |
|
| 987 | 1059 |
ftypes.add(ftype); |
| 988 | 1060 |
store.setFeatureTypes(ftypes, ftype); |
| 989 | 1061 |
|
| 990 | 1062 |
taskStatus.terminate(); |
| 991 |
} catch(Exception ex) {
|
|
| 1063 |
} catch (Exception ex) {
|
|
| 992 | 1064 |
int lineno = 0; |
| 993 |
if( reader!=null ) {
|
|
| 1065 |
if (reader != null) {
|
|
| 994 | 1066 |
lineno = reader.getLine(); |
| 995 | 1067 |
} |
| 996 |
throw new RuntimeException("Problems reading file '"+getFullFileName()+"' near line "+lineno+".", ex);
|
|
| 1068 |
throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex);
|
|
| 997 | 1069 |
|
| 998 | 1070 |
} finally {
|
| 999 |
if ( reader != null ) {
|
|
| 1071 |
if (reader != null) {
|
|
| 1000 | 1072 |
try {
|
| 1001 | 1073 |
reader.close(); |
| 1002 | 1074 |
} catch (Exception ex) {
|
| ... | ... | |
| 1004 | 1076 |
} |
| 1005 | 1077 |
reader = null; |
| 1006 | 1078 |
} |
| 1007 |
if ( in != null ) {
|
|
| 1079 |
if (in != null) {
|
|
| 1008 | 1080 |
try {
|
| 1009 | 1081 |
in.close(); |
| 1010 | 1082 |
} catch (Exception ex) {
|
| ... | ... | |
| 1015 | 1087 |
} |
| 1016 | 1088 |
} |
| 1017 | 1089 |
|
| 1018 |
private int[] automaticDetectionOfTypes() throws IOException {
|
|
| 1090 |
private static class PossibleDataType {
|
|
| 1091 |
|
|
| 1092 |
public boolean possibleInt = true; |
|
| 1093 |
public boolean possibleFloat = true; |
|
| 1094 |
public boolean possibleDouble = true; |
|
| 1095 |
public boolean possibleLong = true; |
|
| 1096 |
public boolean possibleURL = true; |
|
| 1097 |
public boolean possibleDate = true; |
|
| 1098 |
public boolean possibleGeometry = true; |
|
| 1099 |
} |
|
| 1100 |
|
|
| 1101 |
private int[] automaticDetectionOfTypes(String[] headers) throws IOException {
|
|
| 1019 | 1102 |
boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters()); |
| 1020 |
if ( !automatic_types_detection ) {
|
|
| 1103 |
if (!automatic_types_detection) {
|
|
| 1021 | 1104 |
return null; |
| 1022 | 1105 |
} |
| 1023 |
|
|
| 1024 |
final int T_INT = 0; |
|
| 1025 |
final int T_FLOAT = 1; |
|
| 1026 |
final int T_DOUBLE = 2; |
|
| 1027 |
final int T_LONG = 3; |
|
| 1028 |
final int T_URL = 4; |
|
| 1029 |
final int T_DATE = 5; |
|
| 1030 |
boolean possibleDataTypes[][] = null; |
|
| 1031 |
Locale locale = null; |
|
| 1106 |
List<PossibleDataType> possibleDataTypes; |
|
| 1107 |
Locale locale; |
|
| 1032 | 1108 |
int[] types = null; |
| 1033 | 1109 |
|
| 1034 | 1110 |
FileReader in = null; |
| 1035 | 1111 |
SimpleReader reader = null; |
| 1036 |
String headers[] = null; |
|
| 1037 |
SimpleDateFormat dateFormat = new SimpleDateFormat(); |
|
| 1038 | 1112 |
|
| 1039 | 1113 |
try {
|
| 1040 |
|
|
| 1041 | 1114 |
in = new FileReader(this.getCSVParameters().getFile()); |
| 1042 |
|
|
| 1043 | 1115 |
reader = getSimpleReader(in); |
| 1044 |
headers = reader.getHeader(); |
|
| 1045 |
if ( headers == null ) {
|
|
| 1046 |
headers = CSVStoreParameters.getHeaders(getCSVParameters()); |
|
| 1116 |
if (CSVStoreParameters.isFirstLineHeader(getCSVParameters())) {
|
|
| 1117 |
reader.read(); |
|
| 1047 | 1118 |
} |
| 1048 |
types = new int[headers.length]; |
|
| 1049 |
|
|
| 1050 |
possibleDataTypes = new boolean[headers.length][6]; |
|
| 1051 |
for ( int i = 0; i < possibleDataTypes.length; i++ ) {
|
|
| 1052 |
for ( int j = 0; j < 4; j++ ) {
|
|
| 1053 |
possibleDataTypes[i][j] = true; |
|
| 1054 |
} |
|
| 1119 |
possibleDataTypes = new ArrayList<>(headers.length); |
|
| 1120 |
for (String header : headers) {
|
|
| 1121 |
possibleDataTypes.add(new PossibleDataType()); |
|
| 1055 | 1122 |
} |
| 1056 | 1123 |
locale = CSVStoreParameters.getLocale(getCSVParameters()); |
| 1057 |
if ( locale == null ) {
|
|
| 1124 |
if (locale == null) {
|
|
| 1058 | 1125 |
locale = Locale.getDefault(); |
| 1059 | 1126 |
} |
| 1060 |
|
|
| 1061 | 1127 |
DataTypesManager typeManager = ToolsLocator.getDataTypesManager(); |
| 1062 | 1128 |
CoercionWithLocale toDouble = (CoercionWithLocale) typeManager.getCoercion(DataTypes.DOUBLE); |
| 1063 | 1129 |
CoercionWithLocale toFloat = (CoercionWithLocale) typeManager.getCoercion(DataTypes.FLOAT); |
| 1064 | 1130 |
CoercionWithLocale toDate = (CoercionWithLocale) typeManager.getCoercion(DataTypes.DATE); |
| 1065 |
Coercion toInt = typeManager.getCoercion(DataTypes.INT); |
|
| 1066 |
Coercion toLong = typeManager.getCoercion(DataTypes.LONG); |
|
| 1131 |
CoercionWithLocale toInt = (CoercionWithLocale) typeManager.getCoercion(DataTypes.INT); |
|
| 1132 |
CoercionWithLocale toLong = (CoercionWithLocale) typeManager.getCoercion(DataTypes.LONG); |
|
| 1133 |
Coercion toGeom = typeManager.getCoercion(DataTypes.GEOMETRY); |
|
| 1067 | 1134 |
|
| 1068 | 1135 |
List<String> row = reader.read(); |
| 1069 | 1136 |
|
| 1070 |
while ( row != null ) {
|
|
| 1071 |
for ( int i = 0; i < row.size(); i++ ) {
|
|
| 1072 |
Object rawvalue = row.get(i); |
|
| 1073 |
Object value = null; |
|
| 1074 |
if ( possibleDataTypes[i][T_DOUBLE] ) {
|
|
| 1075 |
try {
|
|
| 1076 |
value = toDouble.coerce(rawvalue, locale); |
|
| 1077 |
possibleDataTypes[i][T_DOUBLE] = true; |
|
| 1078 |
} catch (Exception ex) {
|
|
| 1079 |
possibleDataTypes[i][T_DOUBLE] = false; |
|
| 1080 |
} |
|
| 1137 |
while (row != null) {
|
|
| 1138 |
for (int i = 0; i < row.size(); i++) {
|
|
| 1139 |
while( possibleDataTypes.size()<row.size() ) {
|
|
| 1140 |
possibleDataTypes.add(new PossibleDataType()); |
|
| 1081 | 1141 |
} |
| 1082 |
if ( possibleDataTypes[i][T_FLOAT] ) {
|
|
| 1142 |
String rawvalue = row.get(i); |
|
| 1143 |
PossibleDataType possibleDataType = possibleDataTypes.get(i); |
|
| 1144 |
if (possibleDataType.possibleDouble) {
|
|
| 1083 | 1145 |
try {
|
| 1084 |
value = toFloat.coerce(rawvalue, locale);
|
|
| 1085 |
possibleDataTypes[i][T_FLOAT] = true;
|
|
| 1146 |
toDouble.coerce(rawvalue, locale);
|
|
| 1147 |
possibleDataType.possibleDouble = true;
|
|
| 1086 | 1148 |
} catch (Exception ex) {
|
| 1087 |
possibleDataTypes[i][T_FLOAT] = false;
|
|
| 1149 |
possibleDataType.possibleDouble = false;
|
|
| 1088 | 1150 |
} |
| 1089 | 1151 |
} |
| 1090 |
if ( possibleDataTypes[i][T_LONG] ) {
|
|
| 1152 |
if (possibleDataType.possibleFloat) {
|
|
| 1091 | 1153 |
try {
|
| 1092 |
value = toLong.coerce(rawvalue);
|
|
| 1093 |
possibleDataTypes[i][T_LONG] = true;
|
|
| 1154 |
toFloat.coerce(rawvalue, locale);
|
|
| 1155 |
possibleDataType.possibleFloat = true;
|
|
| 1094 | 1156 |
} catch (Exception ex) {
|
| 1095 |
possibleDataTypes[i][T_LONG] = false;
|
|
| 1157 |
possibleDataType.possibleFloat = false;
|
|
| 1096 | 1158 |
} |
| 1097 | 1159 |
} |
| 1098 |
if ( possibleDataTypes[i][T_INT] ) {
|
|
| 1160 |
if (possibleDataType.possibleLong) {
|
|
| 1161 |
possibleDataType.possibleLong = isValidLong(rawvalue); |
|
| 1162 |
} |
|
| 1163 |
if (possibleDataType.possibleInt) {
|
|
| 1164 |
possibleDataType.possibleInt = isValidInteger(rawvalue); |
|
| 1165 |
} |
|
| 1166 |
if (possibleDataType.possibleDate) {
|
|
| 1099 | 1167 |
try {
|
| 1100 |
value = toInt.coerce(rawvalue);
|
|
| 1101 |
possibleDataTypes[i][T_INT] = true;
|
|
| 1168 |
toDate.coerce(rawvalue, locale);
|
|
| 1169 |
possibleDataType.possibleDate = true;
|
|
| 1102 | 1170 |
} catch (Exception ex) {
|
| 1103 |
possibleDataTypes[i][T_INT] = false;
|
|
| 1171 |
possibleDataType.possibleDate = false;
|
|
| 1104 | 1172 |
} |
| 1105 | 1173 |
} |
| 1106 |
if ( possibleDataTypes[i][T_DATE] ) {
|
|
| 1174 |
if (possibleDataType.possibleURL) {
|
|
| 1107 | 1175 |
try {
|
| 1108 |
value = toDate.coerce(rawvalue, locale);
|
|
| 1109 |
possibleDataTypes[i][T_DATE] = true;
|
|
| 1176 |
new URL((String) rawvalue);
|
|
| 1177 |
possibleDataType.possibleURL = true;
|
|
| 1110 | 1178 |
} catch (Exception ex) {
|
| 1111 |
possibleDataTypes[i][T_DATE] = false;
|
|
| 1179 |
possibleDataType.possibleURL = false;
|
|
| 1112 | 1180 |
} |
| 1113 | 1181 |
} |
| 1114 |
if ( possibleDataTypes[i][T_URL] ) {
|
|
| 1182 |
if (possibleDataType.possibleGeometry) {
|
|
| 1115 | 1183 |
try {
|
| 1116 |
value = new URL((String) rawvalue);
|
|
| 1117 |
possibleDataTypes[i][T_URL] = true;
|
|
| 1184 |
toGeom.coerce((String) rawvalue);
|
|
| 1185 |
possibleDataType.possibleGeometry = true;
|
|
| 1118 | 1186 |
} catch (Exception ex) {
|
| 1119 |
possibleDataTypes[i][T_URL] = false;
|
|
| 1187 |
possibleDataType.possibleGeometry = false;
|
|
| 1120 | 1188 |
} |
| 1121 | 1189 |
} |
| 1122 | 1190 |
} |
| 1123 | 1191 |
row = reader.read(); |
| 1124 | 1192 |
} |
| 1125 |
} catch(Exception ex) {
|
|
| 1193 |
int n = 0; |
|
| 1194 |
types = new int[possibleDataTypes.size()]; |
|
| 1195 |
for (PossibleDataType possibleDataType : possibleDataTypes) {
|
|
| 1196 |
if (possibleDataType.possibleInt) {
|
|
| 1197 |
types[n++] = DataTypes.INT; |
|
| 1198 |
continue; |
|
| 1199 |
} |
|
| 1200 |
if (possibleDataType.possibleLong) {
|
|
| 1201 |
types[n++] = DataTypes.LONG; |
|
| 1202 |
continue; |
|
| 1203 |
} |
|
| 1204 |
if (possibleDataType.possibleFloat) {
|
|
| 1205 |
// Forzamos los float a double para evitar perder precision |
|
| 1206 |
types[n++] = DataTypes.DOUBLE; |
|
| 1207 |
continue; |
|
| 1208 |
} |
|
| 1209 |
if (possibleDataType.possibleDouble) {
|
|
| 1210 |
types[n++] = DataTypes.DOUBLE; |
|
| 1211 |
continue; |
|
| 1212 |
} |
|
| 1213 |
if (possibleDataType.possibleURL) {
|
|
| 1214 |
types[n++] = DataTypes.URL; |
|
| 1215 |
continue; |
|
| 1216 |
} |
|
| 1217 |
if (possibleDataType.possibleDate) {
|
|
| 1218 |
types[n++] = DataTypes.DATE; |
|
| 1219 |
continue; |
|
| 1220 |
} |
|
| 1221 |
if (possibleDataType.possibleGeometry) {
|
|
| 1222 |
types[n++] = DataTypes.GEOMETRY; |
|
| 1223 |
continue; |
|
| 1224 |
} |
|
| 1225 |
types[n++] = DataTypes.STRING; |
|
| 1226 |
} |
|
| 1227 |
} catch (Exception ex) {
|
|
| 1126 | 1228 |
int lineno = 0; |
| 1127 |
if( reader!=null ) {
|
|
| 1229 |
if (reader != null) {
|
|
| 1128 | 1230 |
lineno = reader.getLine(); |
| 1129 | 1231 |
} |
| 1130 |
throw new RuntimeException("Problems reading file '"+getFullFileName()+"' near line "+lineno+".", ex);
|
|
| 1232 |
throw new RuntimeException("Problems reading file '" + getFullFileName() + "' near line " + lineno + ".", ex);
|
|
| 1131 | 1233 |
|
| 1132 | 1234 |
} finally {
|
| 1133 |
if ( reader != null ) {
|
|
| 1134 |
try {
|
|
| 1135 |
reader.close(); |
|
| 1136 |
} catch (Exception ex) {
|
|
| 1137 |
// Do nothing |
|
| 1138 |
} |
|
| 1139 |
reader = null; |
|
| 1235 |
IOUtils.closeQuietly(reader); |
|
| 1236 |
IOUtils.closeQuietly(in); |
|
| 1237 |
} |
|
| 1238 |
return types; |
|
| 1239 |
} |
|
| 1240 |
|
|
| 1241 |
private boolean isValidLong(String s) {
|
|
| 1242 |
if (s == null) {
|
|
| 1243 |
return true; |
|
| 1244 |
} |
|
| 1245 |
s = s.trim().toLowerCase(); |
|
| 1246 |
if (s.isEmpty()) {
|
|
| 1247 |
return true; |
|
| 1248 |
} |
|
| 1249 |
try {
|
|
| 1250 |
if (s.startsWith("0x")) {
|
|
| 1251 |
Long.valueOf(s.substring(2), 16); |
|
| 1252 |
} else {
|
|
| 1253 |
Long.valueOf(s); |
|
| 1140 | 1254 |
} |
| 1141 |
if ( in != null ) {
|
|
| 1142 |
try {
|
|
| 1143 |
in.close(); |
|
| 1144 |
} catch (Exception ex) {
|
|
| 1145 |
// Do nothing |
|
| 1146 |
} |
|
| 1147 |
in = null; |
|
| 1148 |
} |
|
| 1255 |
return true; |
|
| 1256 |
} catch (Exception ex) {
|
|
| 1257 |
return false; |
|
| 1258 |
} |
|
| 1259 |
} |
|
| 1149 | 1260 |
|
| 1261 |
private boolean isValidInteger(String s) {
|
|
| 1262 |
if (s == null) {
|
|
| 1263 |
return true; |
|
| 1150 | 1264 |
} |
| 1151 |
for ( int i = 0; i < possibleDataTypes.length; i++ ) {
|
|
| 1152 |
if ( possibleDataTypes[i][T_INT] ) {
|
|
| 1153 |
types[i] = DataTypes.INT; |
|
| 1154 |
continue; |
|
| 1265 |
s = s.trim().toLowerCase(); |
|
| 1266 |
if (s.isEmpty()) {
|
|
| 1267 |
return true; |
|
| 1268 |
} |
|
| 1269 |
try {
|
|
| 1270 |
if (s.startsWith("0x")) {
|
|
| 1271 |
Integer.valueOf(s.substring(2), 16); |
|
| 1272 |
} else {
|
|
| 1273 |
Integer.valueOf(s); |
|
| 1155 | 1274 |
} |
| 1156 |
if ( possibleDataTypes[i][T_LONG] ) {
|
|
| 1157 |
types[i] = DataTypes.LONG; |
|
| 1158 |
continue; |
|
| 1159 |
} |
|
| 1160 |
if ( possibleDataTypes[i][T_DOUBLE] ) {
|
|
| 1161 |
types[i] = DataTypes.DOUBLE; |
|
| 1162 |
continue; |
|
| 1163 |
} |
|
| 1164 |
if ( possibleDataTypes[i][T_FLOAT] ) {
|
|
| 1165 |
types[i] = DataTypes.FLOAT; |
|
| 1166 |
continue; |
|
| 1167 |
} |
|
| 1168 |
if ( possibleDataTypes[i][T_URL] ) {
|
|
| 1169 |
types[i] = DataTypes.URL; |
|
| 1170 |
continue; |
|
| 1171 |
} |
|
| 1172 |
if ( possibleDataTypes[i][T_DATE] ) {
|
|
| 1173 |
types[i] = DataTypes.DATE; |
|
| 1174 |
continue; |
|
| 1175 |
} |
|
| 1176 |
types[i] = DataTypes.STRING; |
|
| 1275 |
return true; |
|
| 1276 |
} catch (Exception ex) {
|
|
| 1277 |
return false; |
|
| 1177 | 1278 |
} |
| 1178 |
return types; |
|
| 1179 | 1279 |
} |
| 1180 | 1280 |
|
| 1181 | 1281 |
} |
Also available in: Unified diff