svn-gvsig-desktop / trunk / org.gvsig.desktop / org.gvsig.desktop.compat.cdc / org.gvsig.fmap.dal / org.gvsig.fmap.dal.file / org.gvsig.fmap.dal.file.csv / src / main / java / org / gvsig / fmap / dal / store / csv / virtualrows / SuperCSVList.java @ 47638
History | View | Annotate | Download (5.44 KB)
1 |
/*
|
---|---|
2 |
* To change this license header, choose License Headers in Project Properties.
|
3 |
* To change this template file, choose Tools | Templates
|
4 |
* and open the template in the editor.
|
5 |
*/
|
6 |
package org.gvsig.fmap.dal.store.csv.virtualrows; |
7 |
|
8 |
import java.io.BufferedReader; |
9 |
import java.io.File; |
10 |
import java.io.IOException; |
11 |
import java.nio.charset.Charset; |
12 |
import java.util.Date; |
13 |
import java.util.List; |
14 |
import java.util.function.Function; |
15 |
import org.apache.commons.io.FilenameUtils; |
16 |
import org.apache.commons.io.IOUtils; |
17 |
import org.apache.commons.io.input.CloseShieldReader; |
18 |
import org.apache.commons.lang3.ArrayUtils; |
19 |
import org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileIndex; |
20 |
import org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileReader; |
21 |
import static org.gvsig.fmap.dal.store.simplereader.virtualrows.RandomAccessFileReader.FILTER_NONE; |
22 |
import org.gvsig.tools.ToolsLocator; |
23 |
import org.gvsig.tools.library.impl.DefaultLibrariesInitializer; |
24 |
import org.gvsig.tools.observer.Observable; |
25 |
import org.gvsig.tools.task.SimpleTaskStatus; |
26 |
import org.gvsig.tools.task.TaskStatus; |
27 |
import org.gvsig.tools.task.TaskStatusManager; |
28 |
import org.supercsv.io.CsvListReader; |
29 |
import org.supercsv.prefs.CsvPreference; |
30 |
|
31 |
/**
|
32 |
*
|
33 |
* @author jjdelcerro
|
34 |
*/
|
35 |
public class SuperCSVList extends AbstractCSVList { |
36 |
|
37 |
private CsvPreference preferences;
|
38 |
|
39 |
public SuperCSVList(File text, File index, Charset charset) throws IOException { |
40 |
super(text, index, charset);
|
41 |
this.preferences = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;
|
42 |
} |
43 |
|
44 |
public SuperCSVList(RandomAccessFileReader reader, RandomAccessFileIndex index, int skipLines) throws IOException { |
45 |
super(reader, index);
|
46 |
this.preferences = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;
|
47 |
this.skipLines = skipLines;
|
48 |
} |
49 |
|
50 |
public void setPreferences(CsvPreference preferences) { |
51 |
this.preferences = preferences;
|
52 |
} |
53 |
|
54 |
@Override
|
55 |
protected List<String> nextRecord() throws IOException { |
56 |
CsvListReader parser = null;
|
57 |
try {
|
58 |
CloseShieldReader theReader = CloseShieldReader.wrap(this.reader);
|
59 |
parser = new CsvListReader(theReader, this.preferences); |
60 |
List<String> values = parser.read(); |
61 |
return values;
|
62 |
} finally {
|
63 |
IOUtils.closeQuietly(parser); |
64 |
} |
65 |
} |
66 |
|
67 |
public static void main(String[] args) throws Exception { |
68 |
new DefaultLibrariesInitializer().fullInitialize();
|
69 |
|
70 |
String fname;
|
71 |
fname = "/home/fdiaz/Descargas/error_union_tablas/Expedientes_CON_REFCAT_trimmed.csv";
|
72 |
// fname = "/home/fdiaz/Descargas/origen_coordenadas.csv";
|
73 |
// fname = "/home/jjdelcerro/Descargas/test/origen_coordenadas.csv";
|
74 |
// fname = "/home/jjdelcerro/Descargas/test/esp_poblaciones.csv";
|
75 |
// fname = "/home/jjdelcerro/Descargas/test/esp_provincias.csv";
|
76 |
// fname = "/home/jjdelcerro/Descargas/test/sigpac.csv";
|
77 |
|
78 |
File data_file = new File(fname); |
79 |
File index_file = new File(FilenameUtils.removeExtension(data_file.getAbsolutePath()) + ".idx"); |
80 |
|
81 |
final TaskStatusManager taskStatusManager = ToolsLocator.getTaskStatusManager();
|
82 |
taskStatusManager.addObserver((Observable observable, Object notification) -> { |
83 |
TaskStatus status = taskStatusManager.getRunningTaskStatusMostRecent(); |
84 |
// System.out.print("\033[?25l\r");
|
85 |
// if( status!=null && status.isRunning() ) {
|
86 |
// System.out.print("\033[?25l\r");
|
87 |
// System.out.print(status.getTitle()+ " - " + status.getLabel());
|
88 |
// System.out.print("\033[K\033[?12l\033[?25h");
|
89 |
// }
|
90 |
// System.out.flush();
|
91 |
}); |
92 |
SimpleTaskStatus status = taskStatusManager.createDefaultSimpleTaskStatus(data_file.getName()); |
93 |
status.add(); |
94 |
Function<BufferedReader, Integer> numberOfLinesInRecord = (BufferedReader breader) -> { |
95 |
CloseShieldReader theReader = CloseShieldReader.wrap(breader); |
96 |
CsvListReader parser = new CsvListReader(theReader, CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE);
|
97 |
try {
|
98 |
List<String> values = parser.read(); |
99 |
} catch (IOException ex) { |
100 |
return 1; |
101 |
} |
102 |
return parser.getLineNumber();
|
103 |
}; |
104 |
RandomAccessFileReader reader = new RandomAccessFileReader(data_file, "UTF-8"); |
105 |
RandomAccessFileIndex index_lines = reader.createIndexOfLines(index_file, false, FILTER_NONE, null, numberOfLinesInRecord); |
106 |
|
107 |
CSVList csv = new SuperCSVList(reader, index_lines, 0); |
108 |
System.out.println("Lines " + csv.size()); |
109 |
int n = 0; |
110 |
for (List<String> record : csv) { |
111 |
if( n<100 ) { |
112 |
System.out.println(String.format("%6d: %s", n, ArrayUtils.toString(record))); |
113 |
} else if( n==100 ) { |
114 |
System.out.println("More records..."); |
115 |
} |
116 |
n++; |
117 |
} |
118 |
System.out.println("-----------------"); |
119 |
long t1 = new Date().getTime(); |
120 |
for (int i = csv.size() - 1; i >= 0; i--) { |
121 |
List<String> record = csv.get(i); |
122 |
if( i<100 ) { |
123 |
System.out.println(String.format("%6d: %s", i, ArrayUtils.toString(record))); |
124 |
} else if( i==100 ) { |
125 |
System.out.println("More records..."); |
126 |
} |
127 |
} |
128 |
long t2 = new Date().getTime(); |
129 |
System.out.println("Time: " + (t2 - t1) + " ms"); |
130 |
IOUtils.closeQuietly(csv); |
131 |
|
132 |
} |
133 |
|
134 |
} |