Revision 41059
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreProvider.java | ||
|---|---|---|
| 31 | 31 |
import java.util.HashMap; |
| 32 | 32 |
import java.util.Iterator; |
| 33 | 33 |
import java.util.List; |
| 34 |
import java.util.Properties; |
|
| 35 | 34 |
|
| 36 | 35 |
import org.apache.commons.io.FilenameUtils; |
| 37 | 36 |
import org.cresques.cts.IProjection; |
| ... | ... | |
| 49 | 48 |
import org.gvsig.fmap.dal.exception.ValidateDataParametersException; |
| 50 | 49 |
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
| 51 | 50 |
import org.gvsig.fmap.dal.feature.EditableFeatureType; |
| 51 |
import org.gvsig.fmap.dal.feature.Feature; |
|
| 52 | 52 |
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor; |
| 53 | 53 |
import org.gvsig.fmap.dal.feature.FeatureSet; |
| 54 |
import org.gvsig.fmap.dal.feature.FeatureStore; |
|
| 54 | 55 |
import org.gvsig.fmap.dal.feature.FeatureType; |
| 55 | 56 |
import org.gvsig.fmap.dal.feature.exception.PerformEditingException; |
| 56 | 57 |
import org.gvsig.fmap.dal.feature.spi.FeatureProvider; |
| ... | ... | |
| 69 | 70 |
import org.gvsig.fmap.geom.primitive.Envelope; |
| 70 | 71 |
import org.gvsig.fmap.geom.primitive.Point; |
| 71 | 72 |
import org.gvsig.fmap.geom.type.GeometryType; |
| 72 |
import org.gvsig.fmap.geom.type.GeometryTypeNotSupportedException; |
|
| 73 |
import org.gvsig.fmap.geom.type.GeometryTypeNotValidException; |
|
| 74 | 73 |
import org.gvsig.tools.ToolsLocator; |
| 75 | 74 |
import org.gvsig.tools.dataTypes.CoercionException; |
| 76 | 75 |
import org.gvsig.tools.dataTypes.DataTypesManager; |
| ... | ... | |
| 81 | 80 |
import org.gvsig.tools.evaluator.AbstractEvaluator; |
| 82 | 81 |
import org.gvsig.tools.evaluator.EvaluatorData; |
| 83 | 82 |
import org.gvsig.tools.evaluator.EvaluatorException; |
| 83 |
import org.gvsig.tools.exception.BaseException; |
|
| 84 | 84 |
import org.gvsig.tools.exception.NotYetImplemented; |
| 85 | 85 |
import org.gvsig.tools.persistence.PersistentState; |
| 86 | 86 |
import org.gvsig.tools.persistence.exception.PersistenceException; |
| 87 | 87 |
import org.gvsig.tools.task.SimpleTaskStatus; |
| 88 | 88 |
import org.gvsig.tools.task.TaskStatusManager; |
| 89 |
import org.gvsig.tools.visitor.VisitCanceledException; |
|
| 90 |
import org.gvsig.tools.visitor.Visitor; |
|
| 89 | 91 |
import org.slf4j.Logger; |
| 90 | 92 |
import org.slf4j.LoggerFactory; |
| 91 | 93 |
import org.supercsv.comment.CommentStartsWith; |
| ... | ... | |
| 107 | 109 |
|
| 108 | 110 |
private long counterNewsOIDs = 0; |
| 109 | 111 |
private Envelope envelope; |
| 112 |
private boolean need_calculate_envelope = false; |
|
| 110 | 113 |
private SimpleTaskStatus taskStatus; |
| 111 | 114 |
|
| 112 | 115 |
|
| ... | ... | |
| 439 | 442 |
|
| 440 | 443 |
public Envelope getEnvelope() throws DataException {
|
| 441 | 444 |
this.open(); |
| 445 |
if( this.envelope!= null ) {
|
|
| 446 |
return this.envelope; |
|
| 447 |
} |
|
| 448 |
if( !this.need_calculate_envelope ) {
|
|
| 449 |
return null; |
|
| 450 |
} |
|
| 451 |
FeatureStore fs = this.getFeatureStore(); |
|
| 452 |
FeatureType ft = fs.getDefaultFeatureType(); |
|
| 453 |
FeatureAttributeDescriptor fad = ft.getAttributeDescriptor(ft.getDefaultGeometryAttributeIndex()); |
|
| 454 |
|
|
| 455 |
try {
|
|
| 456 |
this.envelope = GeometryLocator.getGeometryManager().createEnvelope(fad.getGeomType().getSubType()); |
|
| 457 |
fs.accept(new Visitor() {
|
|
| 458 |
public void visit(Object obj) throws VisitCanceledException, BaseException {
|
|
| 459 |
Feature f = (Feature) obj; |
|
| 460 |
Geometry geom = f.getDefaultGeometry(); |
|
| 461 |
envelope.add(geom.getEnvelope()); |
|
| 462 |
} |
|
| 463 |
}); |
|
| 464 |
} catch (BaseException e) {
|
|
| 465 |
logger.warn("Can't calculate the envelope of CSV file '"+this.getFullName()+"'.",e);
|
|
| 466 |
this.envelope = null; |
|
| 467 |
} |
|
| 468 |
|
|
| 469 |
this.need_calculate_envelope = false; |
|
| 442 | 470 |
return this.envelope; |
| 443 | 471 |
} |
| 444 | 472 |
|
| ... | ... | |
| 584 | 612 |
types[i] = param_types[i]; |
| 585 | 613 |
} |
| 586 | 614 |
} |
| 615 |
int[] param_sizes = CSVStoreParameters.getFieldSizes(this.getParameters()); |
|
| 587 | 616 |
for(int i=0; i<types.length; i++) {
|
| 588 | 617 |
int fieldType = types[i]; |
| 589 | 618 |
String fieldName = headers[i]; |
| 590 |
fType.add(fieldName, fieldType); |
|
| 619 |
EditableFeatureAttributeDescriptor fad = fType.add(fieldName, fieldType); |
|
| 620 |
if( param_sizes!=null && param_sizes[i]>0 ) {
|
|
| 621 |
fad.setSize(param_sizes[i]); |
|
| 622 |
} |
|
| 591 | 623 |
if( fieldType == DataTypes.GEOMETRY && fType.getDefaultGeometryAttributeName() == null) {
|
| 592 | 624 |
fType.setDefaultGeometryAttributeName(fieldName); |
| 593 | 625 |
} |
| ... | ... | |
| 665 | 697 |
String headers[] = null; |
| 666 | 698 |
FeatureStoreProviderServices store = this.getStoreServices(); |
| 667 | 699 |
|
| 700 |
boolean ignore_errors = CSVStoreParameters.getIgnoreErrors(getCSVParameters()); |
|
| 701 |
|
|
| 668 | 702 |
// Initiaize the CSV parser |
| 669 | 703 |
CsvPreference preferences = getCSVPreferences(); |
| 670 | 704 |
in = new FileReader(this.getCSVParameters().getFile()); |
| ... | ... | |
| 684 | 718 |
} |
| 685 | 719 |
|
| 686 | 720 |
// Initialize the feature types |
| 687 |
FeatureType ftype = this.getFeatureType(headers)
|
|
| 688 |
.getNotEditableCopy();
|
|
| 721 |
EditableFeatureType edftype = this.getFeatureType(headers);
|
|
| 722 |
FeatureType ftype = edftype.getNotEditableCopy();
|
|
| 689 | 723 |
List<FeatureType> ftypes = new ArrayList<FeatureType>(); |
| 690 | 724 |
ftypes.add(ftype); |
| 691 | 725 |
store.setFeatureTypes(ftypes, ftype); |
| 692 | 726 |
|
| 693 | 727 |
Coercion coercion[] = new Coercion[ftype.size()]; |
| 728 |
int sizes[] = new int[ftype.size()]; |
|
| 694 | 729 |
for (int i = 0; i < ftype.size(); i++) {
|
| 695 |
coercion[i] = ftype.getAttributeDescriptor(i).getDataType() |
|
| 696 |
.getCoercion(); |
|
| 730 |
sizes[i] = -1; |
|
| 731 |
FeatureAttributeDescriptor ad = ftype.getAttributeDescriptor(i); |
|
| 732 |
coercion[i] = ad.getDataType().getCoercion(); |
|
| 733 |
if( ad.getDataType().getType() == DataTypes.STRING ) {
|
|
| 734 |
if( ad.getSize() == 0 ) {
|
|
| 735 |
// Es un string y no tiene un size asignado. |
|
| 736 |
// Lo ponemos a cero para calcularlo. |
|
| 737 |
sizes[i] = 0; |
|
| 738 |
} |
|
| 739 |
} |
|
| 697 | 740 |
} |
| 698 |
boolean calculate_envelope = (ftype |
|
| 699 |
.getDefaultGeometryAttributeName() != null); |
|
| 741 |
if( ftype.getDefaultGeometryAttributeName() != null) {
|
|
| 742 |
this.need_calculate_envelope = true; |
|
| 743 |
} |
|
| 700 | 744 |
taskStatus.message("_loading");
|
| 701 | 745 |
int count = 0; |
| 702 | 746 |
|
| 747 |
int count_errors = 0; |
|
| 703 | 748 |
List<String> row = reader.read(); |
| 704 | 749 |
while (row != null) {
|
| 705 | 750 |
taskStatus.setCurValue(++count); |
| 706 | 751 |
FeatureProvider feature = this.createFeatureProvider(ftype); |
| 707 | 752 |
for (int i = 0; i < row.size(); i++) {
|
| 708 |
feature.set(i, coercion[i].coerce(row.get(i)));
|
|
| 709 |
}
|
|
| 710 |
if (calculate_envelope) {
|
|
| 711 |
Geometry geom = feature.getDefaultGeometry();
|
|
| 712 |
if (geom != null) {
|
|
| 713 |
if (envelope == null) {
|
|
| 714 |
envelope = (Envelope) geom.getEnvelope().clone();
|
|
| 715 |
} else {
|
|
| 716 |
envelope.add(geom.getEnvelope());
|
|
| 753 |
Object rawvalue = row.get(i);
|
|
| 754 |
try {
|
|
| 755 |
Object value = coercion[i].coerce(rawvalue);
|
|
| 756 |
feature.set(i, value);
|
|
| 757 |
if (sizes[i] >= 0 && value != null) {
|
|
| 758 |
int x = ((String) value).length();
|
|
| 759 |
if (sizes[i] < x) {
|
|
| 760 |
sizes[i] = x;
|
|
| 761 |
}
|
|
| 717 | 762 |
} |
| 763 |
} catch (RuntimeException ex) {
|
|
| 764 |
if (!ignore_errors) {
|
|
| 765 |
throw ex; |
|
| 766 |
} |
|
| 767 |
if( count_errors++ <10 ) {
|
|
| 768 |
logger.warn("Can't load value of attribute "+i+" in row "+count+".",ex);
|
|
| 769 |
} |
|
| 770 |
if( count_errors==10 ) {
|
|
| 771 |
logger.info("Too many errors, suppress messages.");
|
|
| 772 |
} |
|
| 718 | 773 |
} |
| 719 | 774 |
} |
| 720 | 775 |
this.addFeatureProvider(feature); |
| 721 | 776 |
row = reader.read(); |
| 722 | 777 |
} |
| 778 |
for (int i = 0; i < ftype.size(); i++) {
|
|
| 779 |
if( sizes[i]>0 ) {
|
|
| 780 |
EditableFeatureAttributeDescriptor efad = ((EditableFeatureAttributeDescriptor)edftype.getAttributeDescriptor(i)); |
|
| 781 |
efad.setSize(sizes[i]); |
|
| 782 |
} |
|
| 783 |
} |
|
| 784 |
// Volvemos a asignar al store el featuretype, ya que puede |
|
| 785 |
// haber cambiado. |
|
| 786 |
ftype = edftype.getNotEditableCopy(); |
|
| 787 |
ftypes = new ArrayList<FeatureType>(); |
|
| 788 |
ftypes.add(ftype); |
|
| 789 |
store.setFeatureTypes(ftypes, ftype); |
|
| 790 |
|
|
| 723 | 791 |
taskStatus.terminate(); |
| 724 | 792 |
} finally {
|
| 725 | 793 |
if( reader != null ) {
|
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreParameters.java | ||
|---|---|---|
| 36 | 36 |
import org.gvsig.tools.dataTypes.DataTypesManager; |
| 37 | 37 |
import org.gvsig.tools.dynobject.DelegatedDynObject; |
| 38 | 38 |
import org.gvsig.tools.dynobject.DynObject; |
| 39 |
import org.slf4j.Logger; |
|
| 40 |
import org.slf4j.LoggerFactory; |
|
| 39 | 41 |
import org.supercsv.prefs.CsvPreference; |
| 40 | 42 |
import org.supercsv.quote.AlwaysQuoteMode; |
| 41 | 43 |
import org.supercsv.quote.NormalQuoteMode; |
| ... | ... | |
| 44 | 46 |
public class CSVStoreParameters extends AbstractDataParameters implements |
| 45 | 47 |
DataStoreParameters, FilesystemStoreParameters {
|
| 46 | 48 |
|
| 49 |
private static final Logger logger = LoggerFactory.getLogger(CSVStoreParameters.class); |
|
| 50 |
|
|
| 47 | 51 |
public static final String PARAMETERS_DEFINITION_NAME = "CSVStoreParameters"; |
| 48 | 52 |
|
| 49 | 53 |
private static final String FILE = "file"; |
| 54 |
private static final String IGNOREERRORS = "ignoreErrors"; |
|
| 50 | 55 |
private static final String PROFILE = "profile"; |
| 51 | 56 |
private static final String QUOTEPOLICY = "quotePolicy"; |
| 52 | 57 |
private static final String QUOTECHAR = "quoteCharacter"; |
| ... | ... | |
| 222 | 227 |
|
| 223 | 228 |
private static String getDelimiter(String line) {
|
| 224 | 229 |
String sep = null; |
| 225 |
String seps = ",;:-|@#/+$%&!"; |
|
| 230 |
// Cuiaddo con los ":", los he puesto al final a proposito |
|
| 231 |
// ya que podian estar en la cadena para separar el size |
|
| 232 |
// size de cada tipo. |
|
| 233 |
String seps = ",;-|@#/+$%&!:"; |
|
| 226 | 234 |
for( int i=0; i<seps.length(); i ++) {
|
| 227 | 235 |
sep = seps.substring(i, 1); |
| 228 | 236 |
if( line.contains(seps.substring(i, 1))) {
|
| ... | ... | |
| 258 | 266 |
return b.booleanValue(); |
| 259 | 267 |
} |
| 260 | 268 |
|
| 269 |
static boolean getIgnoreErrors(DynObject dynobj) {
|
|
| 270 |
Boolean b = (Boolean) dynobj.getDynValue(IGNOREERRORS); |
|
| 271 |
if( b==null ) {
|
|
| 272 |
return false; |
|
| 273 |
} |
|
| 274 |
return b.booleanValue(); |
|
| 275 |
} |
|
| 276 |
|
|
| 261 | 277 |
// static boolean getIgnoreEmptyLines(DynObject dynobj) {
|
| 262 | 278 |
// Boolean b = (Boolean) dynobj.getDynValue(IGNOREEMPTYLINES); |
| 263 | 279 |
// return b.booleanValue(); |
| ... | ... | |
| 276 | 292 |
String fieldTypeNames[] = s.split("["+sep+"]");
|
| 277 | 293 |
int fieldTypes[] = new int[fieldTypeNames.length]; |
| 278 | 294 |
for( int i=0; i<fieldTypeNames.length; i++ ) {
|
| 279 |
fieldTypes[i] = dataTypeManager.getType(fieldTypeNames[i].trim()); |
|
| 295 |
s = fieldTypeNames[i].trim(); |
|
| 296 |
if( s.contains(":") ) {
|
|
| 297 |
s = s.split(":")[0];
|
|
| 298 |
} |
|
| 299 |
fieldTypes[i] = dataTypeManager.getType(s); |
|
| 280 | 300 |
} |
| 281 | 301 |
return fieldTypes; |
| 282 | 302 |
} |
| 283 | 303 |
|
| 304 |
static int[] getFieldSizes(DynObject dynobj) {
|
|
| 305 |
String s = (String) dynobj.getDynValue(FIELDTYPES); |
|
| 306 |
if( isEmpty(s) ) {
|
|
| 307 |
return null; |
|
| 308 |
} |
|
| 309 |
String sep = getDelimiter(s); |
|
| 310 |
if( sep == null ) {
|
|
| 311 |
return null; |
|
| 312 |
} |
|
| 313 |
String fieldTypeNames[] = s.split("["+sep+"]");
|
|
| 314 |
int fieldSizes[] = new int[fieldTypeNames.length]; |
|
| 315 |
for( int i=0; i<fieldTypeNames.length; i++ ) {
|
|
| 316 |
String fieldtype = fieldTypeNames[i].trim(); |
|
| 317 |
if( fieldtype .contains(":") ) {
|
|
| 318 |
try {
|
|
| 319 |
s = fieldtype .split(":")[1];
|
|
| 320 |
fieldSizes[i] = Integer.parseInt(s); |
|
| 321 |
} catch(Exception ex) {
|
|
| 322 |
logger.warn("Can't get size of field "+i+" ("+fieldtype +").",ex);
|
|
| 323 |
} |
|
| 324 |
} else {
|
|
| 325 |
fieldSizes[i] = 0; |
|
| 326 |
} |
|
| 327 |
} |
|
| 328 |
return fieldSizes; |
|
| 329 |
} |
|
| 330 |
|
|
| 284 | 331 |
static private boolean isEmpty(String s) {
|
| 285 | 332 |
if( s == null ) {
|
| 286 | 333 |
return true; |
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/resources/org/gvsig/fmap/dal/store/csv/CSVParameters.xml | ||
|---|---|---|
| 60 | 60 |
<field name="point" type="string" mandatory="false" defaultValue="" group="Advanced"> |
| 61 | 61 |
<description>This field allow to add a Geometry column to the table. It is a list of field names separated by commas used as dimensions for the attribute GEOM of type point to add to the table.</description> |
| 62 | 62 |
</field> |
| 63 |
<field name="ignoreErrors" type="boolean" mandatory="false" defaultValue="false" group="Advanced"> |
|
| 64 |
<description>Set to true to ignore errors during load of data.</description> |
|
| 65 |
</field> |
|
| 63 | 66 |
</fields> |
| 64 | 67 |
</class> |
| 65 | 68 |
</classes> |
Also available in: Unified diff