Revision 47638
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/resources/META-INF/services/org.gvsig.tools.library.Library | ||
---|---|---|
1 |
org.gvsig.fmap.dal.store.csv.CSVLibrary |
|
1 |
org.gvsig.fmap.dal.store.simplereader.SimpleReaderLibrary |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/json/simplereaders/JSonReader.java | ||
---|---|---|
1 |
/* |
|
2 |
* To change this license theHeader, choose License Headers in Project Properties. |
|
3 |
* To change this template file, choose Tools | Templates |
|
4 |
* and open the template in the editor. |
|
5 |
*/ |
|
6 |
package org.gvsig.fmap.dal.store.json.simplereaders; |
|
7 |
|
|
8 |
import java.io.IOException; |
|
9 |
import java.io.Reader; |
|
10 |
import java.util.ArrayList; |
|
11 |
import java.util.List; |
|
12 |
import javax.json.JsonArray; |
|
13 |
import javax.json.JsonObject; |
|
14 |
import javax.json.JsonString; |
|
15 |
import javax.json.JsonValue; |
|
16 |
import org.apache.commons.io.IOUtils; |
|
17 |
import org.apache.commons.lang3.StringUtils; |
|
18 |
import org.gvsig.fmap.dal.store.csv.CSVStoreParameters; |
|
19 |
import static org.gvsig.fmap.dal.store.csv.CSVStoreParameters.FIRST_LINE_HEADER; |
|
20 |
import static org.gvsig.fmap.dal.store.csv.CSVStoreParameters.HEADER; |
|
21 |
import org.gvsig.fmap.dal.store.simplereader.simplereaders.SimpleReader; |
|
22 |
import org.gvsig.json.Json; |
|
23 |
import org.slf4j.Logger; |
|
24 |
import org.slf4j.LoggerFactory; |
|
25 |
|
|
26 |
/** |
|
27 |
* |
|
28 |
* @author jovivas |
|
29 |
*/ |
|
30 |
public class JSonReader implements SimpleReader { |
|
31 |
|
|
32 |
private static final Logger LOGGER = LoggerFactory.getLogger(JSonReader.class); |
|
33 |
|
|
34 |
private JsonArray jsonData; |
|
35 |
// private final CSVStoreParameters parameters; |
|
36 |
// private List<String> nextLine; |
|
37 |
private int columns; |
|
38 |
private String[] header; |
|
39 |
private int currentElement = 0; |
|
40 |
private final CSVStoreParameters parameters; |
|
41 |
|
|
42 |
public JSonReader(Reader reader, CSVStoreParameters csvParameters) throws IOException { |
|
43 |
String input = IOUtils.toString(reader); |
|
44 |
this.jsonData = Json.createArray(input); |
|
45 |
this.columns = -1; |
|
46 |
this.header = null; |
|
47 |
this.parameters = csvParameters; |
|
48 |
this.parameters.setDynValue(FIRST_LINE_HEADER, false); |
|
49 |
if (StringUtils.isBlank((CharSequence) this.parameters.getDynValue(HEADER))){ |
|
50 |
this.parameters.setDynValue(HEADER, StringUtils.join(this.getHeader(), ',')); |
|
51 |
} |
|
52 |
} |
|
53 |
|
|
54 |
@Override |
|
55 |
public String[] getHeader() throws IOException { |
|
56 |
if (this.header == null) { |
|
57 |
JsonObject firstItem = this.jsonData.getJsonObject(0); |
|
58 |
String[] theHeader = new String[firstItem.size()]; |
|
59 |
int i = 0; |
|
60 |
for (String item : firstItem.keySet()) { |
|
61 |
theHeader[i++] = item; |
|
62 |
} |
|
63 |
this.header = theHeader; |
|
64 |
} |
|
65 |
return this.header; |
|
66 |
} |
|
67 |
|
|
68 |
@Override |
|
69 |
public int getColumnsCount() throws IOException { |
|
70 |
if (this.columns <= 0) { |
|
71 |
this.columns = this.getHeader().length; |
|
72 |
} |
|
73 |
return this.columns; |
|
74 |
} |
|
75 |
|
|
76 |
@Override |
|
77 |
public List<String> read() throws IOException { |
|
78 |
List<String> values = this.read(currentElement); |
|
79 |
if (values == null) { |
|
80 |
return null; |
|
81 |
} |
|
82 |
currentElement += 1; |
|
83 |
return values; |
|
84 |
} |
|
85 |
|
|
86 |
public List<String> read(int rowNumber) throws IOException { |
|
87 |
List<String> values = new ArrayList<>(); |
|
88 |
if (rowNumber < jsonData.size()) { |
|
89 |
JsonObject feature = jsonData.getJsonObject(rowNumber); |
|
90 |
for (String columName : this.getHeader()) { |
|
91 |
if (feature.isNull(columName)) { |
|
92 |
values.add(""); |
|
93 |
} else { |
|
94 |
JsonValue value = feature.get(columName); |
|
95 |
switch (value.getValueType()) { |
|
96 |
case FALSE: |
|
97 |
values.add("false"); |
|
98 |
break; |
|
99 |
case STRING: |
|
100 |
values.add(((JsonString) value).getString()); |
|
101 |
break; |
|
102 |
case TRUE: |
|
103 |
values.add("true"); |
|
104 |
break; |
|
105 |
case NULL: |
|
106 |
values.add(""); |
|
107 |
break; |
|
108 |
case NUMBER: |
|
109 |
case ARRAY: |
|
110 |
case OBJECT: |
|
111 |
default: |
|
112 |
values.add(value.toString()); |
|
113 |
break; |
|
114 |
} |
|
115 |
} |
|
116 |
} |
|
117 |
return values; |
|
118 |
} |
|
119 |
return null; |
|
120 |
} |
|
121 |
|
|
122 |
@Override |
|
123 |
public void close() throws IOException { |
|
124 |
this.jsonData = null; |
|
125 |
} |
|
126 |
|
|
127 |
@Override |
|
128 |
public List<String> skip(int lines) throws IOException { |
|
129 |
this.currentElement += lines; |
|
130 |
if (this.currentElement > this.jsonData.size()) { |
|
131 |
return null; |
|
132 |
} |
|
133 |
return read(this.currentElement); |
|
134 |
} |
|
135 |
|
|
136 |
@Override |
|
137 |
public int getLine() { |
|
138 |
if (this.jsonData == null) { |
|
139 |
return 0; |
|
140 |
} |
|
141 |
return this.currentElement; |
|
142 |
} |
|
143 |
|
|
144 |
@Override |
|
145 |
public List<String> nextRowValues() { |
|
146 |
try { |
|
147 |
return this.read(); |
|
148 |
} catch (IOException ex) { |
|
149 |
throw new RuntimeException(ex); |
|
150 |
} |
|
151 |
} |
|
152 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/AbstractFeatureProviderLoadedOnDemand.java | ||
---|---|---|
1 |
/** |
|
2 |
* gvSIG. Desktop Geographic Information System. |
|
3 |
* |
|
4 |
* Copyright (C) 2007-2013 gvSIG Association. |
|
5 |
* |
|
6 |
* This program is free software; you can redistribute it and/or |
|
7 |
* modify it under the terms of the GNU General Public License |
|
8 |
* as published by the Free Software Foundation; either version 3 |
|
9 |
* of the License, or (at your option) any later version. |
|
10 |
* |
|
11 |
* This program is distributed in the hope that it will be useful, |
|
12 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
13 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
14 |
* GNU General Public License for more details. |
|
15 |
* |
|
16 |
* You should have received a copy of the GNU General Public License |
|
17 |
* along with this program; if not, write to the Free Software |
|
18 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|
19 |
* MA 02110-1301, USA. |
|
20 |
* |
|
21 |
* For any additional information, do not hesitate to contact us |
|
22 |
* at info AT gvsig.com, or visit our website www.gvsig.com. |
|
23 |
*/ |
|
24 |
package org.gvsig.fmap.dal.store.csv; |
|
25 |
|
|
26 |
import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor; |
|
27 |
import org.gvsig.fmap.dal.feature.FeatureType; |
|
28 |
import org.gvsig.fmap.dal.feature.spi.DefaultFeatureProvider; |
|
29 |
import org.gvsig.fmap.dal.feature.spi.FeatureProvider; |
|
30 |
import org.gvsig.fmap.geom.Geometry; |
|
31 |
import org.gvsig.fmap.geom.primitive.Envelope; |
|
32 |
|
|
33 |
public abstract class AbstractFeatureProviderLoadedOnDemand extends DefaultFeatureProvider { |
|
34 |
|
|
35 |
protected boolean loading; |
|
36 |
protected boolean loaded; |
|
37 |
protected String geomName; |
|
38 |
|
|
39 |
public AbstractFeatureProviderLoadedOnDemand(FeatureType type) { |
|
40 |
super(type); |
|
41 |
loading = false; |
|
42 |
loaded = false; |
|
43 |
FeatureAttributeDescriptor geomdesc = type.getDefaultGeometryAttribute(); |
|
44 |
if( geomdesc!=null ) { |
|
45 |
geomName = geomdesc.getName(); |
|
46 |
} |
|
47 |
} |
|
48 |
|
|
49 |
protected void load() { |
|
50 |
if (loading || loaded || this.isNew()) { |
|
51 |
return; |
|
52 |
} |
|
53 |
loading = true; |
|
54 |
try { |
|
55 |
doLoad(); |
|
56 |
} catch (Exception e) { |
|
57 |
} finally { |
|
58 |
loading = false; |
|
59 |
loaded = true; |
|
60 |
} |
|
61 |
} |
|
62 |
|
|
63 |
protected abstract void doLoad(); |
|
64 |
|
|
65 |
@Override |
|
66 |
public void set(int i, Object value) { |
|
67 |
this.load(); |
|
68 |
super.set(i, value); |
|
69 |
} |
|
70 |
|
|
71 |
@Override |
|
72 |
public void set(String name, Object value) { |
|
73 |
this.load(); |
|
74 |
super.set(featureType.getIndex(name), value); |
|
75 |
} |
|
76 |
|
|
77 |
@Override |
|
78 |
public Object get(int i) { |
|
79 |
this.load(); |
|
80 |
return super.get(i); |
|
81 |
} |
|
82 |
|
|
83 |
@Override |
|
84 |
public Object get(String name) { |
|
85 |
this.load(); |
|
86 |
return super.get(name); |
|
87 |
} |
|
88 |
|
|
89 |
@Override |
|
90 |
public Geometry getDefaultGeometry() { |
|
91 |
if( geomName == null ) { |
|
92 |
return null; |
|
93 |
} |
|
94 |
return (Geometry) this.get(geomName); |
|
95 |
} |
|
96 |
|
|
97 |
@Override |
|
98 |
public Envelope getDefaultEnvelope() { |
|
99 |
Geometry geom = this.getDefaultGeometry(); |
|
100 |
if( geom==null ) { |
|
101 |
return null; |
|
102 |
} |
|
103 |
try { |
|
104 |
return geom.getEnvelope(); |
|
105 |
} catch(Exception ex) { |
|
106 |
// En ocasiones con geometrias incorrectas getEnvelop falla. |
|
107 |
return null; |
|
108 |
} |
|
109 |
} |
|
110 |
|
|
111 |
@Override |
|
112 |
public void setOID(Object oid) { |
|
113 |
this.loaded = false; |
|
114 |
super.setOID(oid); |
|
115 |
} |
|
116 |
|
|
117 |
@Override |
|
118 |
public FeatureProvider getCopy() { |
|
119 |
this.load(); |
|
120 |
return super.getCopy(); |
|
121 |
} |
|
122 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVUtils.java | ||
---|---|---|
1 |
/** |
|
2 |
* gvSIG. Desktop Geographic Information System. |
|
3 |
* |
|
4 |
* Copyright (C) 2007-2013 gvSIG Association. |
|
5 |
* |
|
6 |
* This program is free software; you can redistribute it and/or |
|
7 |
* modify it under the terms of the GNU General Public License |
|
8 |
* as published by the Free Software Foundation; either version 3 |
|
9 |
* of the License, or (at your option) any later version. |
|
10 |
* |
|
11 |
* This program is distributed in the hope that it will be useful, |
|
12 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
13 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
14 |
* GNU General Public License for more details. |
|
15 |
* |
|
16 |
* You should have received a copy of the GNU General Public License |
|
17 |
* along with this program; if not, write to the Free Software |
|
18 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|
19 |
* MA 02110-1301, USA. |
|
20 |
* |
|
21 |
* For any additional information, do not hesitate to contact us |
|
22 |
* at info AT gvsig.com, or visit our website www.gvsig.com. |
|
23 |
*/ |
|
24 |
|
|
25 |
package org.gvsig.fmap.dal.store.csv; |
|
26 |
|
|
27 |
import java.io.File; |
|
28 |
import java.io.FileInputStream; |
|
29 |
import java.io.FileNotFoundException; |
|
30 |
import java.io.FileReader; |
|
31 |
import java.io.IOException; |
|
32 |
import java.io.InputStream; |
|
33 |
import java.io.InputStreamReader; |
|
34 |
import java.io.Reader; |
|
35 |
import java.nio.charset.Charset; |
|
36 |
import java.util.Locale; |
|
37 |
import java.util.Map; |
|
38 |
import org.apache.commons.io.FilenameUtils; |
|
39 |
import org.apache.commons.io.IOUtils; |
|
40 |
import org.apache.commons.io.input.BOMInputStream; |
|
41 |
import org.apache.commons.lang3.ArrayUtils; |
|
42 |
import org.apache.commons.lang3.StringUtils; |
|
43 |
import org.gvsig.fmap.dal.DataTypes; |
|
44 |
import org.gvsig.fmap.dal.feature.EditableFeatureAttributeDescriptor; |
|
45 |
import org.gvsig.fmap.dal.feature.EditableFeatureType; |
|
46 |
import static org.gvsig.fmap.dal.store.csv.CSVStoreProvider.NAME; |
|
47 |
import org.gvsig.fmap.dal.store.csv.simplereaders.CSVReaderSuperCSV; |
|
48 |
import org.gvsig.fmap.dal.store.csv.simplereaders.FixedLenReader; |
|
49 |
import org.gvsig.fmap.dal.store.csv.simplereaders.GMLReader; |
|
50 |
import org.gvsig.fmap.dal.store.csv.simplereaders.JSonReader; |
|
51 |
import org.gvsig.fmap.dal.store.simplereader.AutomaticDetectionOfTypes; |
|
52 |
import org.gvsig.fmap.dal.store.simplereader.FieldTypeParser; |
|
53 |
import org.gvsig.fmap.dal.store.simplereader.SimpleReader; |
|
54 |
import org.gvsig.fmap.geom.Geometry; |
|
55 |
import org.gvsig.fmap.geom.GeometryLocator; |
|
56 |
import org.gvsig.fmap.geom.GeometryManager; |
|
57 |
import org.gvsig.fmap.geom.type.GeometryType; |
|
58 |
import org.gvsig.tools.dynobject.Tags; |
|
59 |
import org.gvsig.tools.task.SimpleTaskStatus; |
|
60 |
import org.slf4j.Logger; |
|
61 |
import org.slf4j.LoggerFactory; |
|
62 |
|
|
63 |
/** |
|
64 |
* |
|
65 |
* @author gvSIG Team |
|
66 |
*/ |
|
67 |
@SuppressWarnings("UseSpecificCatch") |
|
68 |
public class CSVUtils { |
|
69 |
|
|
70 |
private static final Logger LOGGER = LoggerFactory.getLogger(CSVUtils.class); |
|
71 |
|
|
72 |
private CSVUtils() { |
|
73 |
|
|
74 |
} |
|
75 |
public static InputStreamReader openFile(File f, String charsetName) throws FileNotFoundException { |
|
76 |
String fullFileName = f==null? "NULL":f.getAbsolutePath(); |
|
77 |
Charset charset = Charset.defaultCharset(); |
|
78 |
InputStream fis = new BOMInputStream(new FileInputStream(f)); |
|
79 |
if (StringUtils.isNotBlank(charsetName)) { |
|
80 |
if (Charset.isSupported(charsetName)) { |
|
81 |
try { |
|
82 |
charset = Charset.forName(charsetName); |
|
83 |
} catch (Throwable th) { |
|
84 |
LOGGER.warn("Can't use charset '" + charsetName + "' for read csv '" + fullFileName + "'.", th); |
|
85 |
} |
|
86 |
} else { |
|
87 |
LOGGER.warn("charset '" + charsetName + "' not supported for read csv '" + fullFileName + "'."); |
|
88 |
} |
|
89 |
} |
|
90 |
InputStreamReader isr = new InputStreamReader(fis, charset); |
|
91 |
return isr; |
|
92 |
} |
|
93 |
|
|
94 |
public static boolean loadFeatureType(CSVStoreParameters parameters, EditableFeatureType featureType, boolean detectTypes, SimpleTaskStatus status) throws IOException { |
|
95 |
InputStreamReader in = null; |
|
96 |
SimpleReader reader = null; |
|
97 |
try { |
|
98 |
String headers[]; |
|
99 |
|
|
100 |
in = openFile( |
|
101 |
parameters.getFile(), |
|
102 |
CSVStoreParameters.getCharset(parameters) |
|
103 |
); |
|
104 |
|
|
105 |
reader = getSimpleReader(parameters, in); |
|
106 |
|
|
107 |
headers = CSVStoreParameters.getHeaders(parameters); |
|
108 |
if (headers == null) { |
|
109 |
if (CSVStoreParameters.isFirstLineHeader(parameters)) { |
|
110 |
headers = reader.getHeader(); |
|
111 |
if (headers == null) { |
|
112 |
if (CSVStoreParameters.getIgnoreErrors(parameters)) { |
|
113 |
headers = getFixedHeaders(reader.getColumnsCount()); |
|
114 |
} else { |
|
115 |
String msg = "Can't retrieve header from csv file '" |
|
116 |
+ parameters.getFile() |
|
117 |
.getAbsolutePath() |
|
118 |
+ "' and not specified in the parameters."; |
|
119 |
LOGGER.warn(msg); |
|
120 |
throw new RuntimeException(msg); |
|
121 |
} |
|
122 |
} |
|
123 |
} else { |
|
124 |
headers = getFixedHeaders(reader.getColumnsCount()); |
|
125 |
} |
|
126 |
} else { |
|
127 |
if (CSVStoreParameters.isFirstLineHeader(parameters)) { |
|
128 |
reader.getHeader(); // Skip and ignore the header of file |
|
129 |
} |
|
130 |
} |
|
131 |
|
|
132 |
AutomaticDetectionOfTypes.DetectedValue[] detectedTypes = null; |
|
133 |
if( detectTypes ) { |
|
134 |
detectedTypes = automaticDetectionOfTypes(parameters, headers, status); |
|
135 |
} |
|
136 |
if( StringUtils.isBlank(headers[headers.length-1]) && |
|
137 |
(detectedTypes==null || detectedTypes[headers.length-1].isBlank()) ) { |
|
138 |
headers = ArrayUtils.remove(headers, headers.length-1); |
|
139 |
} |
|
140 |
if (detectedTypes != null && detectedTypes.length > headers.length) { |
|
141 |
// Se han detectado mas columnas que las que hay en la cabezera, |
|
142 |
// a?adimos mas columnas a la cabezera. |
|
143 |
String[] headers2 = new String[detectedTypes.length]; |
|
144 |
for (int i = 0; i < headers2.length; i++) { |
|
145 |
if (i < headers.length) { |
|
146 |
headers2[i] = headers[i]; |
|
147 |
} else { |
|
148 |
headers2[i] = getFixedHeader(i); |
|
149 |
} |
|
150 |
} |
|
151 |
headers = headers2; |
|
152 |
} |
|
153 |
for (int i = 0; i < headers.length; i++) { |
|
154 |
if (StringUtils.isBlank(headers[i])) { |
|
155 |
headers[i] = getFixedHeader(i); |
|
156 |
} |
|
157 |
} |
|
158 |
// Initialize the feature types |
|
159 |
return fillFeatureType(parameters, featureType, headers, detectedTypes); |
|
160 |
} finally { |
|
161 |
IOUtils.closeQuietly(in); |
|
162 |
IOUtils.closeQuietly(reader); |
|
163 |
} |
|
164 |
} |
|
165 |
public static SimpleReader getSimpleReader(CSVStoreParameters parameters, Reader in) throws IOException { |
|
166 |
SimpleReader reader; |
|
167 |
String filename = CSVStoreParameters.getFileName(parameters); |
|
168 |
if (FilenameUtils.isExtension(filename, "json")){ |
|
169 |
reader= new JSonReader(in,parameters); |
|
170 |
} else if (FilenameUtils.isExtension(filename, "gml")){ |
|
171 |
reader= new GMLReader(in,parameters); |
|
172 |
} else if (CSVStoreParameters.getRawFieldsDefinition(parameters) != null) { |
|
173 |
reader = new FixedLenReader(in, parameters); |
|
174 |
} else { |
|
175 |
reader = new CSVReaderSuperCSV(in, parameters); |
|
176 |
} |
|
177 |
return reader; |
|
178 |
} |
|
179 |
|
|
180 |
private static String getFixedHeader(int column) { |
|
181 |
char[] header = new char[3]; |
|
182 |
|
|
183 |
String s = String.format("%03d", column); |
|
184 |
header[0] = (char) (s.charAt(0) + 17); |
|
185 |
header[1] = (char) (s.charAt(1) + 17); |
|
186 |
header[2] = (char) (s.charAt(2) + 17); |
|
187 |
return String.valueOf(header); |
|
188 |
} |
|
189 |
|
|
190 |
private static String[] getFixedHeaders(int count) { |
|
191 |
String[] headers = new String[count]; |
|
192 |
for (int i = 0; i < headers.length; i++) { |
|
193 |
headers[i] = getFixedHeader(i); |
|
194 |
} |
|
195 |
return headers; |
|
196 |
} |
|
197 |
|
|
198 |
private static AutomaticDetectionOfTypes.DetectedValue[] automaticDetectionOfTypes(CSVStoreParameters parameters, String[] headers, SimpleTaskStatus status) throws IOException { |
|
199 |
String fullFileName = parameters.getFile()==null? "NULL":parameters.getFile().getAbsolutePath(); |
|
200 |
boolean automatic_types_detection = CSVStoreParameters.getAutomaticTypesDetection(parameters); |
|
201 |
if (!automatic_types_detection) { |
|
202 |
return null; |
|
203 |
} |
|
204 |
AutomaticDetectionOfTypes.DetectedValue[] types = null; |
|
205 |
|
|
206 |
Reader in = null; |
|
207 |
SimpleReader reader = null; |
|
208 |
|
|
209 |
try { |
|
210 |
in = openFile( |
|
211 |
parameters.getFile(), |
|
212 |
CSVStoreParameters.getCharset(parameters) |
|
213 |
); |
|
214 |
reader = getSimpleReader(parameters, in); |
|
215 |
AutomaticDetectionOfTypes x = new AutomaticDetectionOfTypes( |
|
216 |
fullFileName |
|
217 |
); |
|
218 |
types = x.detect( |
|
219 |
headers.length, |
|
220 |
reader, |
|
221 |
CSVStoreParameters.isFirstLineHeader(parameters), |
|
222 |
CSVStoreParameters.getLocale(parameters), |
|
223 |
status |
|
224 |
); |
|
225 |
} catch (Exception ex) { |
|
226 |
int lineno = 0; |
|
227 |
if (reader != null) { |
|
228 |
lineno = reader.getLine(); |
|
229 |
} |
|
230 |
throw new RuntimeException("Problems reading file '" + fullFileName + "' near line " + lineno + ".", ex); |
|
231 |
|
|
232 |
} finally { |
|
233 |
IOUtils.closeQuietly(reader); |
|
234 |
IOUtils.closeQuietly(in); |
|
235 |
} |
|
236 |
return types; |
|
237 |
} |
|
238 |
|
|
239 |
private static boolean fillFeatureType(CSVStoreParameters parameters, EditableFeatureType fType, String headers[], AutomaticDetectionOfTypes.DetectedValue automaticTypes[]) { |
|
240 |
String fullFileName = parameters.getFile()==null? "":parameters.getFile().getAbsolutePath(); |
|
241 |
String providerName = NAME; |
|
242 |
|
|
243 |
fType.setHasOID(true); |
|
244 |
|
|
245 |
|
|
246 |
FieldTypeParser[] fieldTypes = new FieldTypeParser[headers.length]; |
|
247 |
// |
|
248 |
// Calculamos cuales pueden ser los tipos de datos |
|
249 |
// |
|
250 |
for (int i = 0; i < fieldTypes.length; i++) { |
|
251 |
fieldTypes[i] = new FieldTypeParser(providerName, fullFileName); |
|
252 |
} |
|
253 |
|
|
254 |
// Asuminos los tipos pasados por parametro, que se supone |
|
255 |
// son los detectados automaticamente. |
|
256 |
if (automaticTypes != null) { |
|
257 |
for (int i = 0; i < fieldTypes.length && i < automaticTypes.length; i++) { |
|
258 |
fieldTypes[i].detectedValue = automaticTypes[i]; |
|
259 |
fieldTypes[i].type = automaticTypes[i].getType(); |
|
260 |
} |
|
261 |
} |
|
262 |
// Luego probamos con lo que diga las cabezeras del CVS, sobreescribiendo |
|
263 |
// los tipos anteriores en caso de definirse en la cabezara. |
|
264 |
boolean all_fields_declare_type = true; |
|
265 |
for (int i = 0; i < fieldTypes.length; i++) { |
|
266 |
if (!fieldTypes[i].parse(headers[i])) { |
|
267 |
LOGGER.warn("Can't parse header of field "+i+ "( "+headers[i]+") in '"+providerName+"' file '" + fullFileName + "'."); |
|
268 |
} |
|
269 |
if( fieldTypes[i].type == DataTypes.UNKNOWN ) { |
|
270 |
all_fields_declare_type = false; |
|
271 |
fieldTypes[i].type = DataTypes.STRING; |
|
272 |
} |
|
273 |
} |
|
274 |
|
|
275 |
// Y por ultimo hacemos caso a lo que se haya especificado en los parametros |
|
276 |
// de apertura del CSV, teniendo esto prioridad sobre todo. |
|
277 |
String param_types_def = CSVStoreParameters.getRawFieldTypes(parameters); |
|
278 |
if (StringUtils.isNotBlank(param_types_def)) { |
|
279 |
String sep = CSVStoreParameters.getDelimiter(param_types_def); |
|
280 |
if (StringUtils.isNotBlank(sep)) { |
|
281 |
String[] param_types = param_types_def.split(sep); |
|
282 |
FieldTypeParser parser = new FieldTypeParser(providerName, fullFileName); |
|
283 |
for (String param_type : param_types) { |
|
284 |
parser.clear(); |
|
285 |
parser.parse(param_type); |
|
286 |
for (FieldTypeParser fieldType : fieldTypes) { |
|
287 |
if (StringUtils.equalsIgnoreCase(fieldType.name, parser.name)) { |
|
288 |
fieldType.copyFrom(parser); |
|
289 |
break; |
|
290 |
} |
|
291 |
} |
|
292 |
} |
|
293 |
} |
|
294 |
} |
|
295 |
// |
|
296 |
// Una vez ya sabemos los tipos de datos rellenamos el feature-type |
|
297 |
// |
|
298 |
Tags ftypeTags = fType.getTags(); |
|
299 |
for (FieldTypeParser fieldType : fieldTypes) { |
|
300 |
EditableFeatureAttributeDescriptor fad = fType.add(fieldType.name, fieldType.type); |
|
301 |
if( fieldType.detectedValue!=null ) { |
|
302 |
fad.setDisplaySize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size)); |
|
303 |
fad.setSize(Math.max(fieldType.detectedValue.getDisplaySize(), fieldType.size)); |
|
304 |
if( fad.getPrecision()<fieldType.detectedValue.getPrecision() ) { |
|
305 |
fad.setPrecision(fieldType.detectedValue.getPrecision()); |
|
306 |
} |
|
307 |
if( fad.getScale()<fieldType.detectedValue.getScale()) { |
|
308 |
fad.setScale(fieldType.detectedValue.getScale()); |
|
309 |
} |
|
310 |
} else { |
|
311 |
fad.setDisplaySize(fieldType.size); |
|
312 |
} |
|
313 |
if (fieldType.type == DataTypes.GEOMETRY ) { |
|
314 |
fad.setGeometryType(fieldType.geomType, fieldType.geomSubtype); |
|
315 |
if( fType.getDefaultGeometryAttributeName() == null ) { |
|
316 |
fType.setDefaultGeometryAttributeName(fieldType.name); |
|
317 |
} |
|
318 |
} |
|
319 |
Locale locale = null; |
|
320 |
if (fieldType.type == DataTypes.TIMESTAMP ) { |
|
321 |
if(!CSVStoreParameters.isBlankOrDefaultLocale(parameters)){ |
|
322 |
locale = CSVStoreParameters.getLocale(parameters); |
|
323 |
} |
|
324 |
} else { |
|
325 |
locale = CSVStoreParameters.getLocale(parameters); |
|
326 |
} |
|
327 |
fad.setLocale(locale); |
|
328 |
for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
|
329 |
try { |
|
330 |
switch(entry.getKey().toLowerCase()) { |
|
331 |
case "expression": |
|
332 |
// Los campos calculados los procesamos en una segunda |
|
333 |
// pasada, cuando ya estan definidos el resto de los campos |
|
334 |
// ya que pueden requerir campos que aun no se han definido. |
|
335 |
break; |
|
336 |
default: |
|
337 |
fad.set(entry.getKey(), entry.getValue()); |
|
338 |
} |
|
339 |
} catch (Exception ex) { |
|
340 |
LOGGER.warn("Can't set property '"+entry.getKey()+"' of '"+fad.getName()+"'.", ex); |
|
341 |
} |
|
342 |
} |
|
343 |
Tags tags = fad.getTags(); |
|
344 |
for (Map.Entry<String, String> entry : fieldType.tags.entrySet()) { |
|
345 |
tags.set(entry.getKey(), entry.getValue()); |
|
346 |
} |
|
347 |
for (Map.Entry<String, String> entry : fieldType.typetags.entrySet()) { |
|
348 |
ftypeTags.set(entry.getKey(), entry.getValue()); |
|
349 |
} |
|
350 |
for (Map.Entry<String, String> entry : fieldType.typeAssignments.entrySet()) { |
|
351 |
try { |
|
352 |
fType.set(entry.getKey(), entry.getValue()); |
|
353 |
} catch(Exception ex) { |
|
354 |
LOGGER.warn("Can't set attribute '"+entry.getKey()+"' in the feature type.", ex); |
|
355 |
} |
|
356 |
} |
|
357 |
} |
|
358 |
// Processamos ahora los campos calculados |
|
359 |
for (FieldTypeParser fieldType : fieldTypes) { |
|
360 |
EditableFeatureAttributeDescriptor fad = fType.getEditableAttributeDescriptor(fieldType.name); |
|
361 |
for (Map.Entry<String, String> entry : fieldType.assignments.entrySet()) { |
|
362 |
try { |
|
363 |
switch(entry.getKey().toLowerCase()) { |
|
364 |
case "expression": |
|
365 |
fad.set(entry.getKey(), entry.getValue()); |
|
366 |
break; |
|
367 |
} |
|
368 |
} catch (Exception ex) { |
|
369 |
LOGGER.warn("Can't set property '"+entry.getKey()+"' in '"+fad.getName()+"' of '"+fullFileName+"'.", ex); |
|
370 |
} |
|
371 |
} |
|
372 |
} |
|
373 |
String[] pointDimensionNames = CSVStoreParameters.getPointDimensionNames(parameters); |
|
374 |
if ( pointDimensionNames != null ) { |
|
375 |
CSVPointAttributeEmulator emulator = new CSVPointAttributeEmulator(pointDimensionNames); |
|
376 |
String columnName = CSVStoreParameters.getPointColumnName(parameters); |
|
377 |
if( StringUtils.isBlank(columnName) ) { |
|
378 |
columnName = "geom"; |
|
379 |
} |
|
380 |
EditableFeatureAttributeDescriptor attr = fType.add(columnName, DataTypes.GEOMETRY, emulator); |
|
381 |
GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
|
382 |
GeometryType gt; |
|
383 |
try { |
|
384 |
if ( emulator.getFieldNames() != null && emulator.getFieldNames().length <= 2 ) { |
|
385 |
gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM2D); |
|
386 |
} else { |
|
387 |
gt = geommgr.getGeometryType(Geometry.TYPES.GEOMETRY, Geometry.SUBTYPES.GEOM3D); |
|
388 |
} |
|
389 |
attr.setGeometryType(gt); |
|
390 |
} catch (Exception e) { |
|
391 |
LOGGER.warn("Can't set geometry type for the calculated field in '"+providerName+"' file '" + fullFileName + "'.", e); |
|
392 |
} |
|
393 |
} |
|
394 |
|
|
395 |
String geometry_column = CSVStoreParameters.getGeometryColumn(parameters); |
|
396 |
if (!StringUtils.isEmpty(geometry_column)) { |
|
397 |
EditableFeatureAttributeDescriptor attr = (EditableFeatureAttributeDescriptor) fType.get(geometry_column); |
|
398 |
if (attr != null ) { |
|
399 |
if( attr.getType() != DataTypes.GEOMETRY ) { |
|
400 |
attr.setDataType(DataTypes.GEOMETRY); |
|
401 |
} |
|
402 |
GeometryManager geommgr = GeometryLocator.getGeometryManager(); |
|
403 |
GeometryType gt; |
|
404 |
try { |
|
405 |
gt = geommgr.getGeometryType( |
|
406 |
CSVStoreParameters.getGeometryType(parameters), |
|
407 |
CSVStoreParameters.getGeometrySubType(parameters) |
|
408 |
); |
|
409 |
attr.setGeometryType(gt); |
|
410 |
} catch (Exception e) { |
|
411 |
LOGGER.warn("Can't set geometry type for the calculated field in CSV file '" + fullFileName + "'.", e); |
|
412 |
} |
|
413 |
fType.setDefaultGeometryAttributeName(geometry_column); |
|
414 |
} |
|
415 |
} |
|
416 |
return all_fields_declare_type; |
|
417 |
} |
|
418 |
|
|
419 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/CSVLibrary.java | ||
---|---|---|
1 |
/** |
|
2 |
* gvSIG. Desktop Geographic Information System. |
|
3 |
* |
|
4 |
* Copyright (C) 2007-2013 gvSIG Association. |
|
5 |
* |
|
6 |
* This program is free software; you can redistribute it and/or |
|
7 |
* modify it under the terms of the GNU General Public License |
|
8 |
* as published by the Free Software Foundation; either version 3 |
|
9 |
* of the License, or (at your option) any later version. |
|
10 |
* |
|
11 |
* This program is distributed in the hope that it will be useful, |
|
12 |
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
13 |
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
14 |
* GNU General Public License for more details. |
|
15 |
* |
|
16 |
* You should have received a copy of the GNU General Public License |
|
17 |
* along with this program; if not, write to the Free Software |
|
18 |
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, |
|
19 |
* MA 02110-1301, USA. |
|
20 |
* |
|
21 |
* For any additional information, do not hesitate to contact us |
|
22 |
* at info AT gvsig.com, or visit our website www.gvsig.com. |
|
23 |
*/ |
|
24 |
package org.gvsig.fmap.dal.store.csv; |
|
25 |
|
|
26 |
import java.util.ArrayList; |
|
27 |
import java.util.List; |
|
28 |
|
|
29 |
import org.gvsig.fmap.dal.DALFileLibrary; |
|
30 |
import org.gvsig.fmap.dal.DALFileLocator; |
|
31 |
import org.gvsig.fmap.dal.DALLibrary; |
|
32 |
import org.gvsig.fmap.dal.DALLocator; |
|
33 |
import org.gvsig.fmap.dal.FileHelper; |
|
34 |
import org.gvsig.fmap.dal.spi.DataManagerProviderServices; |
|
35 |
import org.gvsig.metadata.exceptions.MetadataException; |
|
36 |
import org.gvsig.tools.library.AbstractLibrary; |
|
37 |
import org.gvsig.tools.library.LibraryException; |
|
38 |
|
|
39 |
public class CSVLibrary extends AbstractLibrary { |
|
40 |
|
|
41 |
@Override |
|
42 |
public void doRegistration() { |
|
43 |
registerAsServiceOf(DALLibrary.class); |
|
44 |
require(DALFileLibrary.class); |
|
45 |
} |
|
46 |
|
|
47 |
@Override |
|
48 |
protected void doInitialize() throws LibraryException { |
|
49 |
} |
|
50 |
|
|
51 |
@Override |
|
52 |
protected void doPostInitialize() throws LibraryException { |
|
53 |
List<Throwable> exs = new ArrayList<Throwable>(); |
|
54 |
|
|
55 |
FileHelper.registerParametersDefinition( |
|
56 |
CSVStoreParameters.PARAMETERS_DEFINITION_NAME, |
|
57 |
CSVStoreParameters.class, "CSVParameters.xml"); |
|
58 |
FileHelper.registerParametersDefinition( |
|
59 |
CSVNewStoreParameters.PARAMETERS_DEFINITION_NAME, |
|
60 |
CSVNewStoreParameters.class, "CSVParameters.xml"); |
|
61 |
try { |
|
62 |
|
|
63 |
} catch (Exception e) { |
|
64 |
exs.add(e); |
|
65 |
} |
|
66 |
try { |
|
67 |
FileHelper.registerMetadataDefinition( |
|
68 |
CSVStoreProvider.METADATA_DEFINITION_NAME, |
|
69 |
CSVStoreProvider.class, "CSVMetadata.xml"); |
|
70 |
} catch (MetadataException e) { |
|
71 |
exs.add(e); |
|
72 |
} |
|
73 |
|
|
74 |
DataManagerProviderServices dataman = (DataManagerProviderServices) DALLocator |
|
75 |
.getDataManager(); |
|
76 |
|
|
77 |
try { |
|
78 |
if (!dataman.getStoreProviders().contains(CSVStoreProvider.NAME)) { |
|
79 |
dataman.registerStoreProviderFactory(new CSVStoreProviderFactory(CSVStoreProvider.NAME, CSVStoreProvider.DESCRIPTION)); |
|
80 |
|
|
81 |
} |
|
82 |
} catch (RuntimeException e) { |
|
83 |
exs.add(e); |
|
84 |
} |
|
85 |
|
|
86 |
try { |
|
87 |
DALFileLocator.getFilesystemServerExplorerManager() |
|
88 |
.registerProvider(CSVStoreProvider.NAME, |
|
89 |
CSVStoreProvider.DESCRIPTION, |
|
90 |
CSVFilesystemServerProvider.class); |
|
91 |
} catch (RuntimeException e) { |
|
92 |
exs.add(e); |
|
93 |
} |
|
94 |
|
|
95 |
if (exs.size() > 0) { |
|
96 |
throw new LibraryException(this.getClass(), exs); |
|
97 |
} |
|
98 |
} |
|
99 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/RecordsFileImpl.java | ||
---|---|---|
1 |
package org.gvsig.fmap.dal.store.csv; |
|
2 |
|
|
3 |
import java.io.File; |
|
4 |
import java.io.IOException; |
|
5 |
import java.io.RandomAccessFile; |
|
6 |
import java.nio.ByteBuffer; |
|
7 |
import java.nio.MappedByteBuffer; |
|
8 |
import java.nio.channels.FileChannel; |
|
9 |
import java.sql.Timestamp; |
|
10 |
import java.util.AbstractList; |
|
11 |
import java.util.ArrayList; |
|
12 |
import java.util.List; |
|
13 |
import org.apache.commons.io.IOUtils; |
|
14 |
import org.gvsig.fmap.dal.store.csv.RecordsFile.Record; |
|
15 |
import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_BYTE; |
|
16 |
import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_STRING; |
|
17 |
import static org.gvsig.fmap.dal.store.csv.RecordsFile.TYPE_TIMESTAMP; |
|
18 |
|
|
19 |
/** |
|
20 |
* |
|
21 |
* @author jjdelcerro |
|
22 |
*/ |
|
23 |
public class RecordsFileImpl |
|
24 |
extends AbstractList<Record> |
|
25 |
implements RecordsFile |
|
26 |
{ |
|
27 |
|
|
28 |
public static class RecordImpl implements Record { |
|
29 |
private final RecordType type; |
|
30 |
private final ByteBuffer buffer; |
|
31 |
|
|
32 |
public RecordImpl(RecordType type) { |
|
33 |
this.type = type; |
|
34 |
this.buffer = ByteBuffer.allocate(this.type.getSize()); |
|
35 |
} |
|
36 |
|
|
37 |
@Override |
|
38 |
public RecordType getType() { |
|
39 |
return this.type; |
|
40 |
} |
|
41 |
|
|
42 |
@Override |
|
43 |
public byte[] getBytes() { |
|
44 |
return this.buffer.array(); |
|
45 |
} |
|
46 |
|
|
47 |
@Override |
|
48 |
public ByteBuffer getBuffer() { |
|
49 |
return this.buffer; |
|
50 |
} |
|
51 |
|
|
52 |
@Override |
|
53 |
public void setBytes(byte[] bytes) { |
|
54 |
this.buffer.position(0); |
|
55 |
this.buffer.put(bytes); |
|
56 |
} |
|
57 |
|
|
58 |
@Override |
|
59 |
public int getInt(int n) { |
|
60 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
61 |
return this.buffer.getInt(); |
|
62 |
} |
|
63 |
|
|
64 |
@Override |
|
65 |
public long getLong(int n) { |
|
66 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
67 |
return this.buffer.getLong(); |
|
68 |
} |
|
69 |
|
|
70 |
@Override |
|
71 |
public double getDouble(int n) { |
|
72 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
73 |
return this.buffer.getDouble(); |
|
74 |
} |
|
75 |
|
|
76 |
@Override |
|
77 |
public Timestamp getTimestamp(int n) { |
|
78 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
79 |
long l = this.buffer.getLong(); |
|
80 |
return new Timestamp(l); |
|
81 |
} |
|
82 |
|
|
83 |
@Override |
|
84 |
public String getString(int n) { |
|
85 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
86 |
short sz = this.buffer.getShort(); |
|
87 |
char[] chars = new char[sz]; |
|
88 |
for (int i = 0; i < sz; i++) { |
|
89 |
chars[i] = this.buffer.getChar(i); |
|
90 |
} |
|
91 |
return new String(chars); |
|
92 |
} |
|
93 |
|
|
94 |
@Override |
|
95 |
public byte[] getBytes(int n) { |
|
96 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
97 |
int sz = this.buffer.getInt(); |
|
98 |
byte[] bytes = new byte[sz]; |
|
99 |
for (int i = 0; i < sz; i++) { |
|
100 |
bytes[i] = this.buffer.get(i); |
|
101 |
} |
|
102 |
return bytes; |
|
103 |
} |
|
104 |
|
|
105 |
@Override |
|
106 |
public int getByte(int n) { |
|
107 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
108 |
return this.buffer.get(); |
|
109 |
} |
|
110 |
|
|
111 |
@Override |
|
112 |
public int getShort(int n) { |
|
113 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
114 |
return this.buffer.getShort(); |
|
115 |
} |
|
116 |
|
|
117 |
@Override |
|
118 |
public void setByte(int n, byte v) { |
|
119 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
120 |
this.buffer.put(v); |
|
121 |
} |
|
122 |
|
|
123 |
@Override |
|
124 |
public void setShort(int n, short v) { |
|
125 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
126 |
this.buffer.putShort(v); |
|
127 |
} |
|
128 |
|
|
129 |
@Override |
|
130 |
public void setInt(int n, int v) { |
|
131 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
132 |
this.buffer.putInt(v); |
|
133 |
} |
|
134 |
|
|
135 |
@Override |
|
136 |
public void setLong(int n, long v) { |
|
137 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
138 |
this.buffer.putLong(v); |
|
139 |
} |
|
140 |
|
|
141 |
@Override |
|
142 |
public void setDouble(int n, double v) { |
|
143 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
144 |
this.buffer.putDouble(v); |
|
145 |
} |
|
146 |
|
|
147 |
@Override |
|
148 |
public void setTimestamp(int n, Timestamp v) { |
|
149 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
150 |
this.buffer.putLong(v.getTime()); |
|
151 |
} |
|
152 |
|
|
153 |
@Override |
|
154 |
public void setString(int n, String v) { |
|
155 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
156 |
this.buffer.putShort((short) v.length()); |
|
157 |
for (int i = 0; i < v.length(); i++) { |
|
158 |
this.buffer.putChar(v.charAt(i)); |
|
159 |
} |
|
160 |
} |
|
161 |
|
|
162 |
@Override |
|
163 |
public void setBytes(int n, byte[] v) { |
|
164 |
this.buffer.position(this.type.getFieldOffset(n)); |
|
165 |
this.buffer.putInt(v.length); |
|
166 |
for (int i = 0; i < v.length; i++) { |
|
167 |
this.buffer.put(v[i]); |
|
168 |
} |
|
169 |
} |
|
170 |
} |
|
171 |
|
|
172 |
|
|
173 |
public static class RecordTypeImpl implements RecordType { |
|
174 |
|
|
175 |
private static class FieldType { |
|
176 |
public byte type; |
|
177 |
public int size; |
|
178 |
public int offset; |
|
179 |
} |
|
180 |
|
|
181 |
private final List<FieldType> fields; |
|
182 |
private int size; |
|
183 |
|
|
184 |
public RecordTypeImpl() { |
|
185 |
this.fields = new ArrayList<>(); |
|
186 |
this.size = -1; |
|
187 |
} |
|
188 |
|
|
189 |
public void add(int type, int size) { |
|
190 |
FieldType field = new FieldType(); |
|
191 |
field.size = size; |
|
192 |
field.type = (byte) type; |
|
193 |
this.size = -1; |
|
194 |
field.offset = this.getSize(); |
|
195 |
this.fields.add(field); |
|
196 |
this.size = -1; |
|
197 |
this.size = this.getSize(); |
|
198 |
} |
|
199 |
|
|
200 |
@Override |
|
201 |
public int getSize() { |
|
202 |
if( this.size<0 ) { |
|
203 |
int sz = 0; |
|
204 |
for (FieldType field : fields) { |
|
205 |
switch(field.type) { |
|
206 |
case TYPE_BYTE: |
|
207 |
sz += 1; |
|
208 |
break; |
|
209 |
case TYPE_SHORT: |
|
210 |
sz += 2; |
|
211 |
break; |
|
212 |
case TYPE_INTEGER: |
|
213 |
sz += 4; |
|
214 |
break; |
|
215 |
case TYPE_TIMESTAMP: |
|
216 |
case TYPE_LONG: |
|
217 |
sz += 8; |
|
218 |
break; |
|
219 |
case TYPE_DOUBLE: |
|
220 |
sz += 8; |
|
221 |
break; |
|
222 |
case TYPE_STRING: |
|
223 |
sz += 2 + (field.size*2); |
|
224 |
break; |
|
225 |
case TYPE_BYTES: |
|
226 |
sz += 4 + field.size; |
|
227 |
break; |
|
228 |
} |
|
229 |
} |
|
230 |
this.size = sz; |
|
231 |
} |
|
232 |
return this.size; |
|
233 |
} |
|
234 |
|
|
235 |
@Override |
|
236 |
public int getFieldCount() { |
|
237 |
return this.fields.size(); |
|
238 |
} |
|
239 |
|
|
240 |
@Override |
|
241 |
public int getFieldType(int n) { |
|
242 |
return this.fields.get(n).type; |
|
243 |
} |
|
244 |
|
|
245 |
@Override |
|
246 |
public int getFieldSize(int n) { |
|
247 |
return this.fields.get(n).size; |
|
248 |
} |
|
249 |
|
|
250 |
@Override |
|
251 |
public int getFieldOffset(int n) { |
|
252 |
return this.fields.get(n).offset; |
|
253 |
} |
|
254 |
|
|
255 |
@Override |
|
256 |
public Record createRecord() { |
|
257 |
this.size = -1; |
|
258 |
this.size = this.getSize(); |
|
259 |
return new RecordImpl(this); |
|
260 |
} |
|
261 |
|
|
262 |
@Override |
|
263 |
public byte[] toBytes() { |
|
264 |
ByteBuffer buffer = ByteBuffer.allocate(this.fields.size()*(1+4)); |
|
265 |
for (FieldType field : fields) { |
|
266 |
buffer.put(field.type); |
|
267 |
buffer.putInt(field.size); |
|
268 |
} |
|
269 |
return buffer.array(); |
|
270 |
} |
|
271 |
|
|
272 |
public static RecordType from(ByteBuffer bytes) { |
|
273 |
RecordTypeImpl recordType = new RecordTypeImpl(); |
|
274 |
|
|
275 |
bytes.position(0); |
|
276 |
int len = bytes.limit()/(1+4); |
|
277 |
for (int i = 0; i < len; i++) { |
|
278 |
int type = bytes.get(); |
|
279 |
int size = bytes.getInt(); |
|
280 |
recordType.add(type, size); |
|
281 |
} |
|
282 |
return recordType; |
|
283 |
} |
|
284 |
|
|
285 |
public static RecordType from(byte[] bytes) { |
|
286 |
ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.length); |
|
287 |
buffer.put(bytes); |
|
288 |
return from(buffer); |
|
289 |
} |
|
290 |
|
|
291 |
} |
|
292 |
|
|
293 |
public static class RecordTypeBuilderImpl implements RecordTypeBuilder { |
|
294 |
|
|
295 |
private final RecordTypeImpl recordType; |
|
296 |
|
|
297 |
public RecordTypeBuilderImpl() { |
|
298 |
this.recordType = new RecordTypeImpl(); |
|
299 |
} |
|
300 |
|
|
301 |
@Override |
|
302 |
public RecordTypeBuilder addbyte() { |
|
303 |
this.recordType.add(TYPE_BYTE, 0); |
|
304 |
return this; |
|
305 |
} |
|
306 |
|
|
307 |
@Override |
|
308 |
public RecordTypeBuilder addShort() { |
|
309 |
this.recordType.add(TYPE_SHORT, 0); |
|
310 |
return this; |
|
311 |
} |
|
312 |
|
|
313 |
@Override |
|
314 |
public RecordTypeBuilder addInteger() { |
|
315 |
this.recordType.add(TYPE_INTEGER, 0); |
|
316 |
return this; |
|
317 |
} |
|
318 |
|
|
319 |
@Override |
|
320 |
public RecordTypeBuilder addLong() { |
|
321 |
this.recordType.add(TYPE_LONG, 0); |
|
322 |
return this; |
|
323 |
} |
|
324 |
|
|
325 |
@Override |
|
326 |
public RecordTypeBuilder addDouble() { |
|
327 |
this.recordType.add(TYPE_DOUBLE, 0); |
|
328 |
return this; |
|
329 |
} |
|
330 |
|
|
331 |
@Override |
|
332 |
public RecordTypeBuilder addTimestamp() { |
|
333 |
this.recordType.add(TYPE_TIMESTAMP, 0); |
|
334 |
return this; |
|
335 |
} |
|
336 |
|
|
337 |
@Override |
|
338 |
public RecordTypeBuilder addString(int size) { |
|
339 |
this.recordType.add(TYPE_STRING, size); |
|
340 |
return this; |
|
341 |
} |
|
342 |
|
|
343 |
@Override |
|
344 |
public RecordTypeBuilder addBytes(int size) { |
|
345 |
this.recordType.add(TYPE_BYTES, size); |
|
346 |
return this; |
|
347 |
} |
|
348 |
|
|
349 |
@Override |
|
350 |
public RecordType build() { |
|
351 |
return this.recordType; |
|
352 |
} |
|
353 |
|
|
354 |
public static RecordTypeBuilder recordTypeBuilder() { |
|
355 |
return new RecordTypeBuilderImpl(); |
|
356 |
} |
|
357 |
|
|
358 |
} |
|
359 |
|
|
360 |
private static final int HEADER_SIZE = 1024; |
|
361 |
|
|
362 |
private RandomAccessFile raf; |
|
363 |
private MappedByteBuffer fileByteBuffer; |
|
364 |
private long sz; |
|
365 |
private int header_size; |
|
366 |
private int type_size; |
|
367 |
private RecordType recordType; |
|
368 |
|
|
369 |
public RecordsFileImpl() { |
|
370 |
|
|
371 |
} |
|
372 |
|
|
373 |
@SuppressWarnings("OverridableMethodCallInConstructor") |
|
374 |
public RecordsFileImpl(File f) throws IOException { |
|
375 |
this.open(f); |
|
376 |
} |
|
377 |
|
|
378 |
@SuppressWarnings("OverridableMethodCallInConstructor") |
|
379 |
public RecordsFileImpl(RandomAccessFile raf) throws IOException { |
|
380 |
this.open(raf); |
|
381 |
} |
|
382 |
|
|
383 |
@Override |
|
384 |
public RecordType getRecordType() { |
|
385 |
return this.recordType; |
|
386 |
} |
|
387 |
|
|
388 |
@Override |
|
389 |
public void open(File f) throws IOException { |
|
390 |
RandomAccessFile theRaf = new RandomAccessFile(f,"r"); |
|
391 |
this.open(theRaf); |
|
392 |
} |
|
393 |
|
|
394 |
@Override |
|
395 |
public void open(RandomAccessFile raf) throws IOException { |
|
396 |
this.raf = raf; |
|
397 |
this.closeByteBuffer(); |
|
398 |
this.raf.seek(0); |
|
399 |
this.raf.readLong(); //For future use |
|
400 |
this.raf.readLong(); //For future use |
|
401 |
this.header_size = (int) this.raf.readLong(); //Header size |
|
402 |
this.type_size = (int) this.raf.readLong(); //Record length |
|
403 |
byte[] recordTypeBytes = new byte[this.type_size]; |
|
404 |
this.raf.read(recordTypeBytes); |
|
405 |
this.recordType = RecordType.from(recordTypeBytes); |
|
406 |
|
|
407 |
|
|
408 |
this.sz = (this.raf.length()-this.header_size) / this.recordType.getSize(); |
|
409 |
// TODO: meter en la cabecera el numero de registros para poder escribir datos al final |
|
410 |
} |
|
411 |
|
|
412 |
@Override |
|
413 |
public void create(File f, RecordType recordType) throws IOException { |
|
414 |
RandomAccessFile theRaf = new RandomAccessFile(f,"rw"); |
|
415 |
this.create(theRaf,recordType); |
|
416 |
} |
|
417 |
|
|
418 |
@Override |
|
419 |
public void create(RandomAccessFile raf, RecordType recordType) throws IOException { |
|
420 |
this.raf = raf; |
|
421 |
this.recordType = recordType; |
|
422 |
this.header_size = HEADER_SIZE; |
|
423 |
this.closeByteBuffer(); |
|
424 |
this.sz = 0; |
|
425 |
byte[] recordTypeBytes = this.recordType.toBytes(); |
|
426 |
this.raf.seek(0); |
|
427 |
this.raf.writeLong(0); //For future use |
|
428 |
this.raf.writeLong(0); //For future use |
|
429 |
this.raf.writeLong(this.header_size); //Header size |
|
430 |
this.raf.writeLong(recordTypeBytes.length); //Record length |
|
431 |
this.raf.write(recordTypeBytes); |
|
432 |
this.raf.setLength(this.header_size); |
|
433 |
} |
|
434 |
|
|
435 |
@Override |
|
436 |
public void close() throws IOException { |
|
437 |
this.closeByteBuffer(); |
|
438 |
IOUtils.closeQuietly(this.raf); |
|
439 |
this.raf = null; |
|
440 |
this.sz = -1; |
|
441 |
} |
|
442 |
|
|
443 |
@Override |
|
444 |
public boolean isOpen() { |
|
445 |
return this.raf!=null; |
|
446 |
} |
|
447 |
|
|
448 |
@Override |
|
449 |
public Record get(int position) { |
|
450 |
position = checkIndex(position); |
|
451 |
Record record = this.recordType.createRecord(); |
|
452 |
this.getByteBuffer().position(this.header_size+(position*this.recordType.getSize())); |
|
453 |
this.getByteBuffer().get(record.getBytes()); |
|
454 |
return record; |
|
455 |
} |
|
456 |
|
|
457 |
@Override |
|
458 |
public Record get64(long position) { |
|
459 |
position = checkIndex(position); |
|
460 |
Record record = this.recordType.createRecord(); |
|
461 |
this.getByteBuffer().position((int) (this.header_size+(position*this.recordType.getSize()))); |
|
462 |
this.getByteBuffer().get(record.getBytes()); |
|
463 |
return record; |
|
464 |
} |
|
465 |
|
|
466 |
@Override |
|
467 |
public int size() { |
|
468 |
return (int) this.sz; |
|
469 |
} |
|
470 |
|
|
471 |
@Override |
|
472 |
public long size64() { |
|
473 |
return this.sz; |
|
474 |
} |
|
475 |
|
|
476 |
@Override |
|
477 |
public Record set(int position, Record record) { |
|
478 |
try { |
|
479 |
closeByteBuffer(); |
|
480 |
position = checkIndex(position); |
|
481 |
this.raf.seek(this.header_size + (position * this.recordType.getSize())); |
|
482 |
this.raf.write(record.getBytes()); |
|
483 |
return record; |
|
484 |
} catch (IOException ex) { |
|
485 |
throw new RuntimeException("Can't add record", ex); |
|
486 |
} |
|
487 |
} |
|
488 |
|
|
489 |
private int checkIndex(long index) { |
|
490 |
if (this.raf == null) { |
|
491 |
throw new IllegalStateException("Index not open"); |
|
492 |
} |
|
493 |
if( index < 0 ) { |
|
494 |
index = ((int)this.sz) + index; |
|
495 |
} |
|
496 |
if( index<0 || index>=this.sz) { |
|
497 |
throw new IllegalArgumentException("Index out of range ("+index+")"); |
|
498 |
} |
|
499 |
return (int) index; |
|
500 |
} |
|
501 |
|
|
502 |
private MappedByteBuffer getByteBuffer() { |
|
503 |
try { |
|
504 |
if (this.fileByteBuffer == null) { |
|
505 |
this.fileByteBuffer = this.raf.getChannel().map( |
|
506 |
FileChannel.MapMode.READ_ONLY, |
|
507 |
0, |
|
508 |
this.raf.length() |
|
509 |
); |
|
510 |
} |
|
511 |
return this.fileByteBuffer; |
|
512 |
} catch (IOException ex) { |
|
513 |
throw new RuntimeException("Can't add record", ex); |
|
514 |
} |
|
515 |
} |
|
516 |
|
|
517 |
private void closeByteBuffer() { |
|
518 |
if(this.fileByteBuffer != null){ |
|
519 |
this.fileByteBuffer.force(); |
|
520 |
this.fileByteBuffer = null; |
|
521 |
} |
|
522 |
} |
|
523 |
|
|
524 |
@Override |
|
525 |
public boolean add(Record record) { |
|
526 |
try { |
|
527 |
this.raf.seek(this.raf.length()); |
|
528 |
this.raf.write(record.getBytes()); |
|
529 |
this.closeByteBuffer(); |
|
530 |
this.sz++; |
|
531 |
return true; |
|
532 |
} catch (IOException ex) { |
|
533 |
throw new RuntimeException("Can't add record", ex); |
|
534 |
} |
|
535 |
} |
|
536 |
} |
trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/TestXML.java | ||
---|---|---|
1 |
package org.gvsig.fmap.dal.store.csv; |
|
2 |
|
|
3 |
import java.io.File; |
|
4 |
import java.io.FileInputStream; |
|
5 |
import java.io.InputStreamReader; |
|
6 |
import java.io.Reader; |
|
7 |
import java.nio.charset.Charset; |
|
8 |
import java.nio.charset.StandardCharsets; |
|
9 |
import java.util.ArrayList; |
|
10 |
import java.util.Collections; |
|
11 |
import java.util.HashMap; |
|
12 |
import java.util.HashSet; |
|
13 |
import java.util.Iterator; |
|
14 |
import java.util.LinkedHashMap; |
|
15 |
import java.util.LinkedHashSet; |
|
16 |
import java.util.List; |
|
17 |
import java.util.Map; |
|
18 |
import java.util.Set; |
|
19 |
import javax.xml.parsers.SAXParser; |
|
20 |
import javax.xml.parsers.SAXParserFactory; |
|
21 |
import org.apache.commons.io.FilenameUtils; |
|
22 |
import org.apache.commons.lang3.StringUtils; |
|
23 |
import org.apache.commons.lang3.mutable.MutableLong; |
|
24 |
import org.gvsig.tools.namestranslator.NamesTranslator; |
|
25 |
import org.xml.sax.Attributes; |
|
26 |
import org.xml.sax.InputSource; |
|
27 |
import org.xml.sax.Locator; |
|
28 |
import org.xml.sax.SAXException; |
|
29 |
import org.xml.sax.helpers.DefaultHandler; |
|
30 |
|
|
31 |
public class TestXML { |
|
32 |
|
|
33 |
private static final String XMLFILE1 = "/home/jjdelcerro/datos/geodata/vector/sigpac/2018/Declaracion.xml"; |
|
34 |
private static final String XMLFILE2 = "/home/jjdelcerro/datos/geodata/vector/ARENA2/quincenas-0/TV_03_2019_01_Q1/victimas.xml"; |
|
35 |
private static final String XMLFILE_BIG = "/home/jjdelcerro/datos/geodata/vector/RSUPAC/2020/BDA_RSU_PAC20_1713052020_001.XML"; |
|
36 |
|
|
37 |
public static void main(String[] args) throws Exception { |
|
38 |
TestXML t = new TestXML(); |
|
39 |
|
|
40 |
List<String> tags = new ArrayList(t.extractTags(XMLFILE_BIG)); |
|
41 |
Collections.sort(tags); |
|
42 |
|
|
43 |
// System.out.println("Tags: -------------------------"); |
|
44 |
// for (String tag : tags) { |
|
45 |
// System.out.println(tag); |
|
46 |
// } |
|
47 |
Map<String,List<String>> tables = new LinkedHashMap<>(); |
|
48 |
Map<String,Set<String>> tables_tmp = new LinkedHashMap<>(); |
|
49 |
for (String tag1 : tags) { |
|
50 |
String tableName = tag1; |
|
51 |
Set fields = new HashSet<>(); |
|
52 |
for (String tag2 : tags) { |
|
53 |
if( tableName.equals(tag2) ) { |
|
54 |
continue; |
|
55 |
} |
|
56 |
if( tag2.startsWith(tableName) ) { |
|
57 |
String fieldName = tag2.substring(tableName.length()+1); |
|
58 |
if( !fieldName.contains("/") ) { |
|
59 |
fields.add(fieldName); |
|
60 |
} |
|
61 |
} |
|
62 |
} |
|
63 |
if( !fields.isEmpty() ) { |
|
64 |
tables_tmp.put(tableName,fields); |
|
65 |
} |
|
66 |
} |
|
67 |
Map<String,String> tableNames = new HashMap<>(); |
|
68 |
// NamesTranslator nt = NamesTranslator.createTrimTranslator(45); |
|
69 |
for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) { |
|
70 |
String fullTableName = entry.getKey(); |
|
71 |
String tableName = FilenameUtils.getBaseName(fullTableName); |
|
72 |
// int i = nt.addSource(fullTableName); |
|
73 |
// tableNames.put(tableName,nt.getTranslation(i)); |
|
74 |
tableNames.put(fullTableName,tableName); |
|
75 |
} |
|
76 |
for (Map.Entry<String, Set<String>> entry : tables_tmp.entrySet()) { |
|
77 |
String tableName = entry.getKey(); |
|
78 |
List<String> fields = new ArrayList<>(); |
|
79 |
String[] fieldkeys = tableName.split("/"); |
|
80 |
for (int i = 0; i < fieldkeys.length; i++) { |
|
81 |
String fieldkey = fieldkeys[i]; |
|
82 |
if( i==fieldkeys.length-1 ) { |
|
83 |
fields.add("$ID_"+fieldkey+", PK"); |
|
84 |
} else { |
|
85 |
fields.add("$ID_"+fieldkey+", FK"); |
|
86 |
} |
|
87 |
} |
|
88 |
for (String fieldName : entry.getValue()) { |
|
89 |
if( tables_tmp.containsKey(tableName+"/"+fieldName) ) { |
|
90 |
fields.add("$ID_"+fieldName+" FK"); |
|
91 |
} else { |
|
92 |
fields.add(fieldName); |
|
93 |
} |
|
94 |
} |
|
95 |
Collections.sort(fields); |
|
96 |
tables.put(tableName, fields); |
|
97 |
} |
|
98 |
|
|
99 |
int n = 1; |
|
100 |
for (Map.Entry<String, List<String>> entry : tables.entrySet()) { |
|
101 |
String tableName = entry.getKey(); |
|
102 |
List<String> fields = entry.getValue(); |
|
103 |
System.out.println("TABLE "+tableNames.get(tableName)+" ("+n+", "+tableName+")"); |
|
104 |
for (String field : fields) { |
|
105 |
System.out.println(" "+field); |
|
106 |
} |
|
107 |
n++; |
|
108 |
} |
|
109 |
|
|
110 |
|
|
111 |
// List<List<String>> records = t.getRecords1( |
|
112 |
// XMLFILE1, |
|
113 |
// "DECLARACION/LINEA_DECLARACION", |
|
114 |
// ListBuilder.create( |
|
115 |
// "DECLARACION/LINEA_DECLARACION/PROV", |
|
116 |
//// "DECLARACION/LINEA_DECLARACION/MUN_INE", |
|
117 |
//// "DECLARACION/LINEA_DECLARACION/MUN_CAT", |
|
118 |
//// "DECLARACION/LINEA_DECLARACION/AGREGADO", |
|
119 |
// "DECLARACION/LINEA_DECLARACION/ZONA", |
|
120 |
// "DECLARACION/LINEA_DECLARACION/POLIGONO", |
|
121 |
// "DECLARACION/LINEA_DECLARACION/PARCELA", |
|
122 |
// "DECLARACION/LINEA_DECLARACION/RECINTO", |
|
123 |
// "DECLARACION/LINEA_DECLARACION/PARCELA_AGRICOLA", |
|
124 |
// "DECLARACION/LINEA_DECLARACION/CULTIVO", |
|
125 |
//// "DECLARACION/LINEA_DECLARACION/WKT", |
|
126 |
// "DECLARACION/LINEA_DECLARACION/DN_SURFACE", |
|
127 |
// "DECLARACION/LINEA_DECLARACION/SUPERFICIE_DECLARADA" |
|
128 |
//// "DECLARACION/LINEA_DECLARACION/FC_ALMENDROS", |
|
129 |
//// "DECLARACION/LINEA_DECLARACION/FC_ALGARROBOS", |
|
130 |
//// "DECLARACION/LINEA_DECLARACION/FC_AVELLANOS", |
|
131 |
//// "DECLARACION/LINEA_DECLARACION/FC_NOGALES", |
|
132 |
//// "DECLARACION/LINEA_DECLARACION/FC_PISTACHOS", |
|
133 |
//// "DECLARACION/LINEA_DECLARACION/FC_TOTAL" |
|
134 |
// ) |
Also available in: Unified diff