Revision 47652 trunk/org.gvsig.desktop/org.gvsig.desktop.compat.cdc/org.gvsig.fmap.dal/org.gvsig.fmap.dal.file/org.gvsig.fmap.dal.file.csv/src/main/java/org/gvsig/fmap/dal/store/csv/simplereaders/CSVReaderSuperCSV.java
| CSVReaderSuperCSV.java | ||
|---|---|---|
| 2 | 2 |
|
| 3 | 3 |
import java.io.BufferedReader; |
| 4 | 4 |
import java.io.File; |
| 5 |
import java.io.FileInputStream; |
|
| 6 |
import java.io.FileNotFoundException; |
|
| 7 |
import java.io.FileReader; |
|
| 5 | 8 |
import java.io.IOException; |
| 9 |
import java.io.InputStreamReader; |
|
| 6 | 10 |
import java.io.Reader; |
| 11 |
import java.io.UnsupportedEncodingException; |
|
| 7 | 12 |
import java.util.List; |
| 8 | 13 |
import java.util.function.Function; |
| 9 | 14 |
import org.apache.commons.io.FilenameUtils; |
| 15 |
import org.apache.commons.io.IOUtils; |
|
| 10 | 16 |
import org.apache.commons.io.input.CloseShieldReader; |
| 11 | 17 |
import org.apache.commons.lang3.StringUtils; |
| 12 | 18 |
import org.apache.commons.text.StringEscapeUtils; |
| ... | ... | |
| 72 | 78 |
} |
| 73 | 79 |
return this.columns; |
| 74 | 80 |
} |
| 81 |
|
|
| 82 |
private boolean hasMultilineRecords(SimpleTaskStatus status) throws FileNotFoundException, UnsupportedEncodingException, IOException {
|
|
| 83 |
FileInputStream fis = null; |
|
| 84 |
InputStreamReader theReader = null; |
|
| 85 |
BufferedReader breader = null; |
|
| 86 |
try {
|
|
| 87 |
CSVStoreParameters params = getParameters(); |
|
| 88 |
File data_file = CSVStoreParameters.getFile(params); |
|
| 89 |
String charset = CSVStoreParameters.getCharset(params); |
|
| 90 |
fis = new FileInputStream(data_file); |
|
| 91 |
theReader = new InputStreamReader(fis, charset); |
|
| 92 |
breader = new BufferedReader(theReader); |
|
| 93 |
CsvListReader parser = new CsvListReader(breader, getCSVPreferences()); |
|
| 94 |
// int firstRecordLine = 0; |
|
| 95 |
while (parser.read() != null) {
|
|
| 96 |
if (parser.getLineNumber() != parser.getRowNumber()) {
|
|
| 97 |
return true; |
|
| 98 |
} |
|
| 99 |
} |
|
| 100 |
return false; |
|
| 101 |
} finally {
|
|
| 102 |
IOUtils.closeQuietly(breader); |
|
| 103 |
IOUtils.closeQuietly(theReader); |
|
| 104 |
IOUtils.closeQuietly(fis); |
|
| 105 |
} |
|
| 106 |
} |
|
| 75 | 107 |
|
| 76 | 108 |
@Override |
| 77 | 109 |
public GetItemWithSize64<List<String>> getVirtualRows(SimpleTaskStatus status) {
|
| ... | ... | |
| 83 | 115 |
if( data_file.length()< 10*1024*1024 ) {
|
| 84 | 116 |
return null; |
| 85 | 117 |
} |
| 86 |
|
|
| 87 |
Function<BufferedReader, Integer> numberOfLinesInRecord = new Function<BufferedReader, Integer>() {
|
|
| 88 |
@Override |
|
| 89 |
public Integer apply(BufferedReader breader) {
|
|
| 90 |
CloseShieldReader theReader = CloseShieldReader.wrap(breader); |
|
| 91 |
CsvListReader parser = new CsvListReader(theReader, getCSVPreferences()); |
|
| 92 |
try {
|
|
| 93 |
List<String> values = parser.read(); |
|
| 94 |
} catch (IOException ex) {
|
|
| 95 |
return 1; |
|
| 118 |
Function<BufferedReader, Integer> numberOfLinesInRecord = null; |
|
| 119 |
if(this.hasMultilineRecords(status)){
|
|
| 120 |
numberOfLinesInRecord = new Function<BufferedReader, Integer>() {
|
|
| 121 |
@Override |
|
| 122 |
public Integer apply(BufferedReader breader) {
|
|
| 123 |
CloseShieldReader theReader = CloseShieldReader.wrap(breader); |
|
| 124 |
CsvListReader parser = new CsvListReader(theReader, getCSVPreferences()); |
|
| 125 |
try {
|
|
| 126 |
List<String> values = parser.read(); |
|
| 127 |
} catch (IOException ex) {
|
|
| 128 |
return 1; |
|
| 129 |
} |
|
| 130 |
return parser.getLineNumber(); |
|
| 96 | 131 |
} |
| 97 |
return parser.getLineNumber(); |
|
| 98 |
} |
|
| 99 |
}; |
|
| 132 |
}; |
|
| 133 |
} |
|
| 100 | 134 |
|
| 101 | 135 |
String charset = CSVStoreParameters.getCharset(params); |
| 102 | 136 |
File index_file = getIndexFile(data_file); |
Also available in: Unified diff