Application: gvSIG desktop

/*

 * To change this license header, choose License Headers in Project Properties.

 * To change this template file, choose Tools | Templates

 * and open the template in the editor.

 */

package org.gvsig.fmap.dal.store.csv;

import java.io.BufferedReader;

import java.io.Closeable;

import java.io.File;

import java.io.IOException;

import java.nio.ByteBuffer;

import java.nio.CharBuffer;

import java.nio.charset.Charset;

import java.nio.charset.StandardCharsets;

import java.util.AbstractList;

import java.util.ArrayList;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Objects;

import javax.xml.parsers.SAXParser;

import javax.xml.parsers.SAXParserFactory;

import org.apache.commons.io.FilenameUtils;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang3.StringUtils;

import org.apache.commons.lang3.tuple.Pair;

import org.gvsig.fmap.dal.store.csv.RecordsFile.Record;

import org.gvsig.fmap.dal.store.csv.RecordsFile.RecordType;

import static org.gvsig.fmap.dal.store.csv.RecordsFile.RecordTypeBuilder.recordTypeBuilder;

import org.gvsig.fmap.dal.store.csv.virtualrows.RandomAccessFileReader;

import org.gvsig.tools.util.GetItemWithSize64;

import org.gvsig.tools.util.ListBuilder;

import org.xml.sax.Attributes;

import org.xml.sax.InputSource;

import org.xml.sax.Locator;

import org.xml.sax.SAXException;

import org.xml.sax.helpers.DefaultHandler;

/**

 *

 * @author jjdelcerro

 */

@SuppressWarnings("UseSpecificCatch")

public class XMLFileAsList

        extends AbstractList<List<String>>

        implements Closeable, GetItemWithSize64<List<String>> {

    private static final int IDXFIELD_LINE = 0;

    private static final int IDXFIELD_COLUMN = 1;

    private static final int IDXFIELD_LINEPOS = 2;

    private static final int IDXFIELD_RECORDPOS = 3;

    private final RandomAccessFileReader reader;

    private final String recordPath;

    private RecordsFile index;

    private List<String> fieldPaths;

    //                                                         ruta   recordnum                       ruta   recordnum

 // public XMLFileAsList(File text, Charset charset, List<Pair<String,Integer>> recordPath, List<Pair<String,Integer>> fieldPaths) throws IOException {

    public XMLFileAsList(File text, Charset charset, String recordPath, List<String> fieldPaths) throws IOException {

        this.reader = new RandomAccessFileReader(text, charset);

        this.index = null;

        this.recordPath = recordPath;

    }

    public XMLFileAsList(File text, File index, Charset charset, String recordPath, List<String> fieldPaths) throws IOException {

        this.reader = new RandomAccessFileReader(text, charset);

        if (index.exists()) {

            // TODO: Force to create index if text newer than index

            this.index = new RecordsFileImpl(index);

        } else {

            this.createIndex(index);

        }

        this.recordPath = recordPath;

    }

    public XMLFileAsList(RandomAccessFileReader reader, RecordsFile index, String recordPath, List<String> fieldPaths) throws IOException {

        this.reader = reader;

        this.index = index;

        this.recordPath = recordPath;

    }

    @Override

    public void close() {

        IOUtils.closeQuietly(this.reader);

        IOUtils.closeQuietly(this.index);

    }

    private List<String> getRecord(long position, List<String> fieldPaths) {

        class StopParserSAXException extends SAXException {

        }

        class ParseRecordsHandler extends DefaultHandler {

            Locator locator;

            List<String> path = new ArrayList<>();

            Map<String, String> record = new HashMap<>();

            StringBuilder value = new StringBuilder();

            List<String> values = new ArrayList<>();

            @Override

            public void setDocumentLocator(Locator locator) {

                this.locator = locator;

            }

            @Override

            public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

                path.add(localName);

                String path_s = StringUtils.join(path, "/");

                if (StringUtils.equalsIgnoreCase(path_s, recordPath)) {

                    return;

                }

                this.value.setLength(0);

//                    this.value.trimToSize();

            }

            @Override

            public void characters(char[] ch, int start, int length) throws SAXException {

                value.append(new String(ch, start, length));

            }

            @Override

            public void endElement(String uri, String localName, String qName) throws SAXException {

                String path_s = StringUtils.join(path, "/");

                if (StringUtils.equalsIgnoreCase(path_s, recordPath)) {

                    for (String fieldPath : fieldPaths) {

                        values.add(record.get(fieldPath));

                    }

                    throw new StopParserSAXException();

                } else {

                    for (String fieldPath : fieldPaths) {

                        if (StringUtils.equalsIgnoreCase(path_s, fieldPath)) {

                            record.put(fieldPath, this.value.toString());

                            // No break to handle repeated columns

                        }

                    }

                }

                path.remove(path.size() - 1);

            }

        }

        ParseRecordsHandler handler = null;

        try {

            Record record = this.index.get64(position);

            long recordPosition = record.getLong(IDXFIELD_RECORDPOS);

            this.reader.seek(recordPosition);

            BufferedReader breader = new BufferedReader(this.reader, 1024 * 8);

            InputSource is = new InputSource(breader);

            SAXParserFactory spf = SAXParserFactory.newInstance();

            spf.setNamespaceAware(true);

            SAXParser saxParser = spf.newSAXParser();

            handler = new ParseRecordsHandler();

            saxParser.parse(is, handler);

        } catch (StopParserSAXException ex) {

            if (handler != null) {

                return handler.values;

            }

        } catch (Exception ex) {

            throw new RuntimeException("Can't parse record " + position, ex);

        }

        return null; // ?????

    }

    @Override

    public List<String> get(int index) {

        return this.getRecord(index, fieldPaths);

    }

    @Override

    public List<String> get64(long position) {

        return this.getRecord(position, fieldPaths);

    }

    @Override

    public int size() {

        return this.index.size();

    }

    @Override

    public long size64() {

        return this.index.size64();

    }

    final public void createIndex(File indexFile) throws IOException {

        try {

            // 1. Creamos el indice vacio

            RecordType recordType = recordTypeBuilder()

                    .addLong() // IDXFIELD_LINE

                    .addLong() // IDXFIELD_COLUMN

                    .addLong() // IDXFIELD_LINEPOS

                    .addLong() // IDXFIELD_RECORDPOS

                    .build();

            Record record = recordType.createRecord();

            final RecordsFile theIndex = new RecordsFileImpl();

            theIndex.create(indexFile, recordType, 0);

            // 2. Rellenamos los campos numero de linea y columna de cada registro 

            //    en el indice.

            // Para ello nos recorremos el XML y vamos escribiendo en el indice.

            this.reader.rewind();

            InputSource is = new InputSource(reader);

            SAXParserFactory spf = SAXParserFactory.newInstance();

            spf.setNamespaceAware(true);

            SAXParser saxParser = spf.newSAXParser();

            List<String> path = new ArrayList<>();

            // TODO: Aqui habria que quedarse con la primera aparicion de

            // los campos solicitados que no pertenezcan al registro principal.

            // Probablemente habria que guardar esos valores en disco junto al

            // fichero de indice.

            saxParser.parse(is, new DefaultHandler() {

                Locator locator;

                @Override

                public void setDocumentLocator(Locator locator) {

                    this.locator = locator;

                }

                @Override

                public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {

                    path.add(localName);

                    String path_s = StringUtils.join(path, "/");

                    if (StringUtils.equalsIgnoreCase(path_s, recordPath)) {

                        record.setLong(IDXFIELD_LINE, locator.getLineNumber() - 1);

                        record.setLong(IDXFIELD_COLUMN, locator.getColumnNumber() - 1);

                        record.setLong(IDXFIELD_LINEPOS, 0);

                        record.setLong(IDXFIELD_RECORDPOS, 0);

                        theIndex.add(record);

                    }

                }

                @Override

                public void endElement(String uri, String localName, String qName) throws SAXException {

                    path.remove(path.size() - 1);

                }

            });

            // 3. Nos recorremos el indice y calculamos la posicion de la linea

            //    para cada registro.

            this.reader.rewind();

            PositionCalculator positionCalculator = new PositionCalculator(reader);

            for (int i = 0; i < theIndex.size(); i++) {

                Record r = theIndex.get(i);

                positionCalculator.next(r.getLong(IDXFIELD_LINE), r.getLong(IDXFIELD_COLUMN));

                r.setLong(IDXFIELD_LINEPOS, positionCalculator.getLinePosition());

                r.setLong(IDXFIELD_RECORDPOS, positionCalculator.getColumnPosition());

                theIndex.set(i, record);

            }

            this.index = new RecordsFileImpl(indexFile);

        } catch (Exception ex) {

            throw new IOException("Can't create index " + Objects.toString(indexFile), ex);

        }

    }

    private static class CharacterSize {

        private final Charset charset;

        private final CharBuffer charBuffer;

        public CharacterSize(Charset charset) {

            this.charset = charset;

            this.charBuffer = CharBuffer.allocate(1);

        }

        public int size(char ch) {

            this.charBuffer.put(0, ch);

            this.charBuffer.position(0);

            ByteBuffer buffer = this.charset.encode(this.charBuffer);

            return buffer.limit();

        }

        public int size(char[] cbuf, int off, int len) {

            CharBuffer cb = CharBuffer.wrap(cbuf, off, len);

            ByteBuffer byteBuffer = this.charset.encode(cb);

            return byteBuffer.limit();

        }

        public int size(char[] cbuf) {

            CharBuffer cb = CharBuffer.wrap(cbuf, 0, cbuf.length);

            ByteBuffer byteBuffer = this.charset.encode(cb);

            return byteBuffer.limit();

        }

        public int size(String s) {

            CharBuffer cb = CharBuffer.wrap(s);

            ByteBuffer byteBuffer = this.charset.encode(cb);

            return byteBuffer.limit();

        }

    }

    private static class PositionCalculator {

        private final RandomAccessFileReader reader;

        private final BufferedReader breader;

        private long currentPosition;

        private long currentColumn;

        private long currentLine;

        private long currentColumnPosition;

        private long currentLinePosition;

        private final char[] ch;

        private final CharacterSize characterSize;

        public PositionCalculator(RandomAccessFileReader reader) {

            this.reader = reader;

            this.breader = new BufferedReader(this.reader, 1024 * 8);

            this.currentPosition = this.reader.getCurrentPosition();

            this.currentLine = 0;

            this.currentColumn = 0;

            this.ch = new char[1];

            this.characterSize = new CharacterSize(this.reader.getCharset());

        }

        public boolean next(long line, long column) throws IOException {

            while (this.currentLine < line) {

                if (breader.read(this.ch, 0, 1) < 1) {

                    return false;

                }

                char c = ch[0];

                this.currentPosition += characterSize.size(c);

                if (c == '\n') {

                    this.currentLine++;

                    this.currentColumn = 0;

                }

            }

            this.currentLinePosition = this.currentPosition;

            while (this.currentColumn < column) {

                if (breader.read(this.ch, 0, 1) < 1) {

                    return false;

                }

                char c = ch[0];

                this.currentPosition += characterSize.size(c);

                if (c == '\n') {

                    // Uff, esto seria un error, ya que la linea corriente no

                    // tiene tantas columnas.

                    throw new IOException("Illegal column number " + column + " for line " + line);

                }

                this.currentColumn++;

            }

            this.currentColumnPosition = this.currentPosition;

            return true;

        }

        public long getLinePosition() {

            return this.currentLinePosition;

        }

        public long getColumnPosition() {

            return this.currentColumnPosition;

        }

    }

    public static void main(String[] args) throws Exception {

        final String XMLFILE1 = "/home/jjdelcerro/datos/geodata/vector/gml/navarra.gml";

        final String XMLFILE2 = "/home/jjdelcerro/datos/geodata/vector/gml/Municipis/Municipis.gml";

        String gmlfile = XMLFILE1;

        XMLFileAsList gml = new XMLFileAsList(

                new File(gmlfile), 

                new File(FilenameUtils.removeExtension(gmlfile)+".gmlidx"), 

                StandardCharsets.UTF_8, 

                "FeatureCollection/featureMember", 

                ListBuilder.create(

                        "FEATURE/fid",

                        "FEATURE/MERINDAD",

                        "FEATURE/Cnt_MERIND",

                        "FEATURE/Nombre"

                )

        );

        System.out.println("File: "+gmlfile);

        for (int i = 0; i < gml.size(); i++) {

            List<String> item = gml.get(i);

            System.out.println(item);

        }

    }

}