Revision 41062
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreParameters.java | ||
|---|---|---|
| 58 | 58 |
private static final String RECORDSEPARATOR = "recordSeparator"; |
| 59 | 59 |
private static final String DELIMITER = "delimiter"; |
| 60 | 60 |
private static final String COMMENTSTARTMARKER = "commentStartMarker"; |
| 61 |
private static final String AUTOMATICTYPESDETECTION = "automaticTypesDetection"; |
|
| 61 | 62 |
|
| 62 | 63 |
//private static final String ESCAPECHARACTER = "escapeCharacter"; |
| 63 | 64 |
|
| ... | ... | |
| 266 | 267 |
return b.booleanValue(); |
| 267 | 268 |
} |
| 268 | 269 |
|
| 270 |
static boolean getAutomaticTypesDetection(DynObject dynobj) {
|
|
| 271 |
Boolean b = (Boolean) dynobj.getDynValue(AUTOMATICTYPESDETECTION); |
|
| 272 |
if( b==null ) {
|
|
| 273 |
return false; |
|
| 274 |
} |
|
| 275 |
return b.booleanValue(); |
|
| 276 |
} |
|
| 277 |
|
|
| 269 | 278 |
static boolean getIgnoreErrors(DynObject dynobj) {
|
| 270 | 279 |
Boolean b = (Boolean) dynobj.getDynValue(IGNOREERRORS); |
| 271 | 280 |
if( b==null ) {
|
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVStoreProvider.java | ||
|---|---|---|
| 27 | 27 |
import java.io.FileReader; |
| 28 | 28 |
import java.io.FileWriter; |
| 29 | 29 |
import java.io.IOException; |
| 30 |
import java.net.URL; |
|
| 31 |
import java.text.SimpleDateFormat; |
|
| 30 | 32 |
import java.util.ArrayList; |
| 31 | 33 |
import java.util.HashMap; |
| 32 | 34 |
import java.util.Iterator; |
| ... | ... | |
| 577 | 579 |
} |
| 578 | 580 |
} |
| 579 | 581 |
|
| 580 |
private EditableFeatureType getFeatureType(String headers[]) {
|
|
| 582 |
private EditableFeatureType getFeatureType(String headers[], int automaticTypes[]) {
|
|
| 581 | 583 |
EditableFeatureType fType = getStoreServices().createFeatureType(this.getName()); |
| 582 | 584 |
fType.setHasOID(true); |
| 583 | 585 |
DataTypesManager dataTypesManager = ToolsLocator.getDataTypesManager(); |
| 584 | 586 |
|
| 585 | 587 |
int[] types = new int[headers.length]; |
| 588 |
// |
|
| 589 |
// Calculamos cuales pueden ser los tipos de datos |
|
| 590 |
// |
|
| 591 |
|
|
| 592 |
// Por defecto todos string |
|
| 593 |
for( int i=0; i<types.length ; i++) {
|
|
| 594 |
types[i] = DataTypes.STRING; |
|
| 595 |
} |
|
| 596 |
|
|
| 597 |
// Luego asuminos los tipos pasados por parametro, que se supone |
|
| 598 |
// son los detectados automaticamente. |
|
| 599 |
if (automaticTypes != null) {
|
|
| 600 |
for (int i = 0; i < types.length && i < automaticTypes.length; i++) {
|
|
| 601 |
types[i] = automaticTypes[i]; |
|
| 602 |
} |
|
| 603 |
} |
|
| 604 |
|
|
| 605 |
// Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo |
|
| 606 |
// los tipos anteriores en caso de definirse en la cabezara. |
|
| 586 | 607 |
for( int i=0; i<types.length; i++) {
|
| 587 | 608 |
String s = headers[i]; |
| 588 | 609 |
String typename = null; |
| ... | ... | |
| 596 | 617 |
headers[i] = ss[0]; |
| 597 | 618 |
typename = ss[1]; |
| 598 | 619 |
types[i] = dataTypesManager.getType(typename); |
| 599 |
} else {
|
|
| 600 |
types[i] = DataTypes.STRING; |
|
| 601 |
typename = "string"; |
|
| 602 | 620 |
} |
| 603 | 621 |
if( types[i]==DataTypes.INVALID ) {
|
| 604 | 622 |
types[i] = DataTypes.STRING; |
| ... | ... | |
| 606 | 624 |
} |
| 607 | 625 |
} |
| 608 | 626 |
|
| 627 |
// Y por ultimo hacemos caso a lo que se haya especificado en los parametros |
|
| 628 |
// de apertura del CSV, teniendo esto prioridad sobre todo. |
|
| 609 | 629 |
int[] param_types = CSVStoreParameters.getFieldTypes(this.getParameters()); |
| 610 | 630 |
if( param_types != null ) {
|
| 611 | 631 |
for( int i=0; i<types.length && i<param_types.length; i++) {
|
| 612 | 632 |
types[i] = param_types[i]; |
| 613 | 633 |
} |
| 614 | 634 |
} |
| 635 |
|
|
| 636 |
// |
|
| 637 |
// Una vez ya sabemos los tipos de datos rellenamos el feature-type |
|
| 638 |
// |
|
| 615 | 639 |
int[] param_sizes = CSVStoreParameters.getFieldSizes(this.getParameters()); |
| 616 | 640 |
for(int i=0; i<types.length; i++) {
|
| 617 | 641 |
int fieldType = types[i]; |
| ... | ... | |
| 693 | 717 |
FileReader in = null; |
| 694 | 718 |
CsvListReader reader = null; |
| 695 | 719 |
try {
|
| 696 |
Envelope envelope = null; |
|
| 697 | 720 |
String headers[] = null; |
| 698 | 721 |
FeatureStoreProviderServices store = this.getStoreServices(); |
| 699 | 722 |
|
| ... | ... | |
| 718 | 741 |
} |
| 719 | 742 |
|
| 720 | 743 |
// Initialize the feature types |
| 721 |
EditableFeatureType edftype = this.getFeatureType(headers); |
|
| 744 |
EditableFeatureType edftype = this.getFeatureType(headers, automaticDetectionOfTypes());
|
|
| 722 | 745 |
FeatureType ftype = edftype.getNotEditableCopy(); |
| 723 | 746 |
List<FeatureType> ftypes = new ArrayList<FeatureType>(); |
| 724 | 747 |
ftypes.add(ftype); |
| ... | ... | |
| 746 | 769 |
|
| 747 | 770 |
int count_errors = 0; |
| 748 | 771 |
List<String> row = reader.read(); |
| 772 |
|
|
| 749 | 773 |
while (row != null) {
|
| 750 | 774 |
taskStatus.setCurValue(++count); |
| 751 | 775 |
FeatureProvider feature = this.createFeatureProvider(ftype); |
| ... | ... | |
| 808 | 832 |
} |
| 809 | 833 |
} |
| 810 | 834 |
} |
| 835 |
|
|
| 836 |
private int[] automaticDetectionOfTypes() throws IOException {
|
|
| 837 |
boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(getCSVParameters()); |
|
| 838 |
if( !automatic_types_detection ) {
|
|
| 839 |
return null; |
|
| 840 |
} |
|
| 841 |
|
|
| 842 |
final int T_INT = 0; |
|
| 843 |
final int T_FLOAT = 1; |
|
| 844 |
final int T_DOUBLE = 2; |
|
| 845 |
final int T_LONG = 3; |
|
| 846 |
final int T_URL = 4; |
|
| 847 |
final int T_DATE = 5; |
|
| 848 |
boolean possibleDataTypes[][] = null; |
|
| 849 |
int[] types = null; |
|
| 811 | 850 |
|
| 851 |
FileReader in = null; |
|
| 852 |
CsvListReader reader = null; |
|
| 853 |
String headers[] = null; |
|
| 854 |
SimpleDateFormat dateFormat = new SimpleDateFormat(); |
|
| 855 |
|
|
| 856 |
try {
|
|
| 857 |
CsvPreference preferences = getCSVPreferences(); |
|
| 858 |
in = new FileReader(this.getCSVParameters().getFile()); |
|
| 859 |
|
|
| 860 |
reader = new CsvListReader(in, preferences); |
|
| 861 |
headers = reader.getHeader(true); |
|
| 862 |
if (headers == null) {
|
|
| 863 |
headers = CSVStoreParameters.getHeaders(getCSVParameters()); |
|
| 864 |
} |
|
| 865 |
types = new int[headers.length]; |
|
| 866 |
|
|
| 867 |
possibleDataTypes = new boolean[headers.length][6]; |
|
| 868 |
for (int i = 0; i < possibleDataTypes.length; i++) {
|
|
| 869 |
for( int j=0; j<4; j++ ) {
|
|
| 870 |
possibleDataTypes[i][j] = true; |
|
| 871 |
} |
|
| 872 |
} |
|
| 873 |
List<String> row = reader.read(); |
|
| 874 |
|
|
| 875 |
while (row != null) {
|
|
| 876 |
for (int i = 0; i < row.size(); i++) {
|
|
| 877 |
Object rawvalue = row.get(i); |
|
| 878 |
Object value = null; |
|
| 879 |
if( possibleDataTypes[i][T_DOUBLE] ) {
|
|
| 880 |
try {
|
|
| 881 |
value = Double.parseDouble((String) rawvalue); |
|
| 882 |
possibleDataTypes[i][T_DOUBLE] = true; |
|
| 883 |
} catch (Exception ex) {
|
|
| 884 |
possibleDataTypes[i][T_DOUBLE] = false; |
|
| 885 |
} |
|
| 886 |
} |
|
| 887 |
if( possibleDataTypes[i][T_FLOAT] ) {
|
|
| 888 |
try {
|
|
| 889 |
value = Float.parseFloat((String) rawvalue); |
|
| 890 |
possibleDataTypes[i][T_FLOAT] = true; |
|
| 891 |
} catch (Exception ex) {
|
|
| 892 |
possibleDataTypes[i][T_FLOAT] = false; |
|
| 893 |
} |
|
| 894 |
} |
|
| 895 |
if (possibleDataTypes[i][T_LONG]) {
|
|
| 896 |
try {
|
|
| 897 |
value = Long.parseLong((String) rawvalue); |
|
| 898 |
possibleDataTypes[i][T_LONG] = true; |
|
| 899 |
} catch (Exception ex) {
|
|
| 900 |
possibleDataTypes[i][T_LONG] = false; |
|
| 901 |
} |
|
| 902 |
} |
|
| 903 |
if (possibleDataTypes[i][T_INT]) {
|
|
| 904 |
try {
|
|
| 905 |
value = Integer.parseInt((String) rawvalue); |
|
| 906 |
possibleDataTypes[i][T_INT] = true; |
|
| 907 |
} catch (Exception ex) {
|
|
| 908 |
possibleDataTypes[i][T_INT] = false; |
|
| 909 |
} |
|
| 910 |
} |
|
| 911 |
if (possibleDataTypes[i][T_DATE]) {
|
|
| 912 |
try {
|
|
| 913 |
value = dateFormat.parse((String) rawvalue); |
|
| 914 |
possibleDataTypes[i][T_DATE] = true; |
|
| 915 |
} catch (Exception ex) {
|
|
| 916 |
possibleDataTypes[i][T_DATE] = false; |
|
| 917 |
} |
|
| 918 |
} |
|
| 919 |
if (possibleDataTypes[i][T_URL]) {
|
|
| 920 |
try {
|
|
| 921 |
value = new URL((String) rawvalue); |
|
| 922 |
possibleDataTypes[i][T_URL] = true; |
|
| 923 |
} catch (Exception ex) {
|
|
| 924 |
possibleDataTypes[i][T_URL] = false; |
|
| 925 |
} |
|
| 926 |
} |
|
| 927 |
} |
|
| 928 |
row = reader.read(); |
|
| 929 |
} |
|
| 930 |
} finally {
|
|
| 931 |
if (reader != null) {
|
|
| 932 |
try {
|
|
| 933 |
reader.close(); |
|
| 934 |
} catch (Exception ex) {
|
|
| 935 |
// Do nothing |
|
| 936 |
} |
|
| 937 |
reader = null; |
|
| 938 |
} |
|
| 939 |
if (in != null) {
|
|
| 940 |
try {
|
|
| 941 |
in.close(); |
|
| 942 |
} catch (Exception ex) {
|
|
| 943 |
// Do nothing |
|
| 944 |
} |
|
| 945 |
in = null; |
|
| 946 |
} |
|
| 947 |
|
|
| 948 |
} |
|
| 949 |
for (int i = 0; i < possibleDataTypes.length ; i++) {
|
|
| 950 |
if( possibleDataTypes[i][T_DOUBLE] ) {
|
|
| 951 |
types[i] = DataTypes.DOUBLE; |
|
| 952 |
continue; |
|
| 953 |
} |
|
| 954 |
if( possibleDataTypes[i][T_FLOAT] ) {
|
|
| 955 |
types[i] = DataTypes.FLOAT; |
|
| 956 |
continue; |
|
| 957 |
} |
|
| 958 |
if( possibleDataTypes[i][T_LONG] ) {
|
|
| 959 |
types[i] = DataTypes.LONG; |
|
| 960 |
continue; |
|
| 961 |
} |
|
| 962 |
if( possibleDataTypes[i][T_INT] ) {
|
|
| 963 |
types[i] = DataTypes.INT; |
|
| 964 |
continue; |
|
| 965 |
} |
|
| 966 |
if( possibleDataTypes[i][T_URL] ) {
|
|
| 967 |
types[i] = DataTypes.URL; |
|
| 968 |
continue; |
|
| 969 |
} |
|
| 970 |
if( possibleDataTypes[i][T_DATE] ) {
|
|
| 971 |
types[i] = DataTypes.DATE; |
|
| 972 |
continue; |
|
| 973 |
} |
|
| 974 |
types[i] = DataTypes.STRING; |
|
| 975 |
} |
|
| 976 |
return types; |
|
| 977 |
} |
|
| 978 |
|
|
| 812 | 979 |
} |
| trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/resources/org/gvsig/fmap/dal/store/csv/CSVParameters.xml | ||
|---|---|---|
| 27 | 27 |
<value label="Quotes fields which contain special characters">NormalQuoteMode</value> |
| 28 | 28 |
</availableValues> |
| 29 | 29 |
</field> |
| 30 |
<field name="recordSeparator" type="string" mandatory="false" defaultValue="" group="Advanced">
|
|
| 30 |
<field name="recordSeparator" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 31 | 31 |
<description>Sets the record separator of the format to the specified character.</description> |
| 32 | 32 |
</field> |
| 33 |
<field name="delimiter" type="string" mandatory="false" defaultValue="" group="Advanced">
|
|
| 33 |
<field name="delimiter" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 34 | 34 |
<description>Sets the field delimiter of the format to the specified character.</description> |
| 35 | 35 |
</field> |
| 36 |
<field name="quoteCharacter" type="string" mandatory="false" defaultValue="" group="Advanced">
|
|
| 36 |
<field name="quoteCharacter" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 37 | 37 |
<description>Sets the quoteChar of the format to the specified character.</description> |
| 38 | 38 |
</field> |
| 39 |
<field name="commentStartMarker" type="string" mandatory="false" defaultValue="" group="Advanced">
|
|
| 39 |
<field name="commentStartMarker" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 40 | 40 |
<description>Sets the comment start marker of the format to the specified character.</description> |
| 41 | 41 |
</field> |
| 42 |
<field name="escapeCharacter" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 42 |
<field name="automaticTypesDetection" type="boolean" mandatory="false" defaultValue="true" group="Advanced"> |
|
| 43 |
<description>This flag activate the autodetection of types in the CSV. Can be slow.</description> |
|
| 44 |
</field> |
|
| 45 |
<field name="escapeCharacter" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 43 | 46 |
<description>Sets the escape character of the format to the specified character</description> |
| 44 | 47 |
</field> |
| 45 | 48 |
<field name="header" type="string" mandatory="false" defaultValue="" group="Advanced"> |
| 46 | 49 |
<description>The header or empty if parsed automatically</description> |
| 47 | 50 |
</field> |
| 48 |
<field name="surroundingSpacesNeedQuotes" type="boolean" mandatory="false" defaultValue="false" group="Advanced">
|
|
| 51 |
<field name="surroundingSpacesNeedQuotes" type="boolean" mandatory="false" defaultValue="false" group="Advanced"> |
|
| 49 | 52 |
<description>the surroundingSpacesNeedQuotes flag.</description> |
| 50 | 53 |
</field> |
| 51 |
<field name="CRS" type="crs" mandatory="false" group="Advanced">
|
|
| 54 |
<field name="CRS" type="crs" mandatory="false" group="Advanced"> |
|
| 52 | 55 |
<description>The coordinate reference system used in this CSV file if contains geometries</description> |
| 53 | 56 |
</field> |
| 54 |
<field name="fieldtypes" type="string" mandatory="false" defaultValue="" group="Advanced">
|
|
| 57 |
<field name="fieldtypes" type="string" mandatory="false" defaultValue="" group="Advanced"> |
|
| 55 | 58 |
<description>Comma separated list of field types. If empty strings are assumed (Eg: integer,string,double,boolean,float,geometry).</description> |
| 56 | 59 |
</field> |
| 57 | 60 |
<field name="charset" type="string" mandatory="false" defaultValue="UTF-8" group="Advanced"> |
| ... | ... | |
| 60 | 63 |
<field name="point" type="string" mandatory="false" defaultValue="" group="Advanced"> |
| 61 | 64 |
<description>This field allow to add a Geometry column to the table. It is a list of field names separated by commas used as dimensions for the attribute GEOM of type point to add to the table.</description> |
| 62 | 65 |
</field> |
| 63 |
<field name="ignoreErrors" type="boolean" mandatory="false" defaultValue="false" group="Advanced">
|
|
| 66 |
<field name="ignoreErrors" type="boolean" mandatory="false" defaultValue="false" group="Advanced"> |
|
| 64 | 67 |
<description>Set to true to ignore errors during load of data.</description> |
| 65 | 68 |
</field> |
| 66 | 69 |
</fields> |
Also available in: Unified diff