Application: gvSIG desktop

/**

 * gvSIG. Desktop Geographic Information System.

 *

 * Copyright (C) 2007-2013 gvSIG Association.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version 3

 * of the License, or (at your option) any later version.

 *

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU General Public License for more details.

 *

 * You should have received a copy of the GNU General Public License

 * along with this program; if not, write to the Free Software

 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,

 * MA  02110-1301, USA.

 *

 * For any additional information, do not hesitate to contact us

 * at info AT gvsig.com, or visit our website www.gvsig.com.

 */

package org.gvsig.fmap.dal.store.dbf.utils;

import java.io.IOException;

import java.nio.BufferOverflowException;

import java.nio.ByteBuffer;

import java.nio.MappedByteBuffer;

import java.nio.channels.FileChannel;

import java.nio.charset.Charset;

import java.text.FieldPosition;

import java.text.NumberFormat;

import java.util.Arrays;

import java.util.Calendar;

import java.util.Date;

import java.util.Iterator;

import java.util.Locale;

import org.apache.commons.lang3.StringUtils;

import org.gvsig.fmap.dal.DataTypes;

import org.gvsig.fmap.dal.exception.CloseException;

import org.gvsig.fmap.dal.exception.InitializeException;

import org.gvsig.fmap.dal.exception.UnsupportedEncodingException;

import org.gvsig.fmap.dal.exception.WriteException;

import org.gvsig.fmap.dal.feature.Feature;

import org.gvsig.fmap.dal.feature.FeatureAttributeDescriptor;

import org.gvsig.fmap.dal.feature.FeatureType;

/**

 * A DbaseFileReader is used to read a dbase III format file. The general use of

 * this class is: <CODE><PRE>

 * DbaseFileHeader header = ...

 * WritableFileChannel out = new FileOutputStream("thefile.dbf").getChannel();

 * DbaseFileWriter w = new DbaseFileWriter(header,out);

 * while ( moreRecords ) {

 *   w.write( getMyRecord() );

 * }

 * w.close();

 * </PRE></CODE> You must supply the <CODE>moreRecords</CODE> and

 * <CODE>getMyRecord()</CODE> logic...

 * 

 * @author Ian Schneider

 */

public class DbaseFileWriter {

    private DbaseFileHeader header;

    private DbaseFileWriter.FieldFormatter formatter =

        new DbaseFileWriter.FieldFormatter();

    FileChannel channel;

    private ByteBuffer buffer;

    private boolean headDrity = false;

    private ByteBuffer blank;

    private int blankSize;

    //private Charset charset = Charset.forName("ISO-8859-1");

    private Charset charset;

    /**

     * Create a DbaseFileWriter using the specified header and writing to the

     * given channel.

     * 

     * @param header

     *            The DbaseFileHeader to write.

     * @param out

     *            The Channel to write to.

     * 

     * 

     * @throws InitializeWriterException

     * @throws IOException

     *             If errors occur while initializing.

     */

    public DbaseFileWriter(DbaseFileHeader header, FileChannel out,

        boolean isNew) throws InitializeException {

        this.header = header;

        this.channel = out;

        this.headDrity = isNew;

        this.setCharset(Charset.forName(header.mappingEncoding(header.getCharsetName())));

        init();

    }

    private void init() throws InitializeException {

        try {

            if (this.channel.size() < this.header.getHeaderLength()) {

                this.writeHeader();

            }

            buffer = ByteBuffer.allocateDirect(header.getRecordLength());

        } catch (Exception e) {

            throw new InitializeException("DBF Writer", e);

        }

    }

    private void write() throws WriteException {

        buffer.position(0);

        int r = buffer.remaining();

        try {

            while ((r -= channel.write(buffer)) > 0) {

                ; // do nothing

            }

        } catch (IOException e) {

            throw new WriteException("DBF Writer", e);

        }

    }

    private void writeHeader() throws WriteException {

        try {

            channel.position(0);

            header.writeHeader(channel);

        } catch (IOException e) {

            throw new WriteException("DBF Writer", e);

        }

    }

    /**

     * Write a single dbase record.

     * 

     * @param record

     *            The entries to write.

     * @throws UnsupportedEncodingException

     * @throws WriteException

     */

    public void append(Feature feature) throws WriteException,

        UnsupportedEncodingException {

        this.fillBuffer(feature);

        try {

            this.moveToEOF();

        } catch (IOException e) {

            throw new WriteException("DbaseFileWriter", e);

        }

        this.header.setNumRecords(this.header.getNumRecords() + 1);

        write();

        this.headDrity = true;

    }

    private void fillBuffer(Feature feature)

        throws UnsupportedEncodingException, WriteException {

        FeatureType featureType = feature.getType();

        try {

            buffer.position(0);

            // put the 'not-deleted' marker

            buffer.put((byte) ' ');

            @SuppressWarnings("unchecked")

            Iterator<FeatureAttributeDescriptor> iterator =

                featureType.iterator();

            while (iterator.hasNext()) {

                FeatureAttributeDescriptor fad = iterator.next();

                if (fad.isComputed()) {

                    continue;

                }

                if (fad.getName().length() > DbaseFile.MAX_FIELD_NAME_LENGTH) {

                    throw new FieldNameTooLongException(

                        "DBF file", fad.getName());

                }

                int type = fad.getType();

                if (type == DataTypes.GEOMETRY) {

                    continue;

                }

                encodeField(fad, feature);

            }

        } catch (Exception e) {

            throw new WriteException("DbaseFileWriter", e);

        }

    }

    private void moveToEOF() throws IOException {

        this.moveTo(this.header.getNumRecords());

    }

    private void moveTo(long numReg) throws IOException {

        // if (!(channel instanceof FileChannel)) {

        // throw new IOException(

        // "DbaseFileWriterNIO: channel is not a FileChannel. Cannot position properly");

        // }

        long newPos =

            header.getHeaderLength() + numReg * header.getRecordLength();

        if (this.channel.position() != newPos) {

            this.channel.position(newPos);

        }

    }

    /**

     * Write a single dbase record. Useful to update a dbf.

     * 

     * @param record

     *            The entries to write.

     * @throws WriteException

     * @throws UnsupportedEncodingException

     */

    public void update(Feature feature, long numReg) throws WriteException,

        UnsupportedEncodingException {

        this.fillBuffer(feature);

        try {

            this.moveTo(numReg);

        } catch (IOException e) {

            throw new WriteException("DbaseFileWriter", e);

        }

        write();

    }

    private String fieldString(FeatureAttributeDescriptor attr, Feature feature) throws java.io.UnsupportedEncodingException {

        int type = attr.getType();

        int dbfFieldIndex = this.header.getFieldIndex(attr.getName());

        final int fieldLen = header.getFieldLength(dbfFieldIndex);

        String fieldString = "";

        //

        // https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm

        //

        if (DataTypes.BOOLEAN == type) {

            boolean b = feature.getBoolean(attr.getIndex());

            if (b) {

                fieldString = "T";

            } else {

                fieldString = "F";

            }

        } else

            if (DataTypes.BYTE == type) {

                fieldString = String.valueOf(feature.getByte(attr.getIndex()));

            } else

                if (DataTypes.DATE == type) {

                    Date date = feature.getDate(attr.getIndex());

                    fieldString = formatter.getFieldString(date);

                } else

                    if (DataTypes.DOUBLE == type) {

                        double d = feature.getDouble(attr.getIndex());

                        fieldString =

                            formatter.getFieldString(fieldLen,

                                header.getFieldDecimalCount(dbfFieldIndex), d);

                    } else

                        if (DataTypes.FLOAT == type) {

                            float f = feature.getFloat(attr.getIndex());

                            fieldString =

                                formatter.getFieldString(fieldLen,

                                    header.getFieldDecimalCount(dbfFieldIndex),

                                    f);

                        } else

                            if (DataTypes.INT == type) {

                                int integer = feature.getInt(attr.getIndex());

                                fieldString =

                                    formatter.getFieldString(fieldLen, header

                                        .getFieldDecimalCount(dbfFieldIndex),

                                        integer);

                            } else

                                if (DataTypes.LONG == type) {

                                    long l = feature.getLong(attr.getIndex());

                                    fieldString =

                                        formatter

                                            .getFieldString(

                                                fieldLen,

                                                header

                                                    .getFieldDecimalCount(dbfFieldIndex),

                                                l);

                                } else

                                    if (DataTypes.STRING == type) {

                                        String s =

                                            feature.getString(attr.getIndex());

                                        return s;

                                    }

        return fieldString;

    }

    private void encodeField(FeatureAttributeDescriptor attr, Feature feature) throws java.io.UnsupportedEncodingException, UnsupportedEncodingException {

        int type = attr.getType();

        int dbfFieldIndex = this.header.getFieldIndex(attr.getName());

        final int fieldLen = header.getFieldLength(dbfFieldIndex);

        String fieldString = "";

        if( DataTypes.BOOLEAN == type ) {

            boolean b = feature.getBoolean(attr.getIndex());

            if( b ) {

                safeEncode("T", 1, true);

            } else {

                safeEncode("F", 1, true);

            }

        } else if( DataTypes.BYTE == type ) {

            fieldString = String.valueOf(feature.getByte(attr.getIndex()));

            safeEncode(fieldString, 8, false);

        } else if( DataTypes.DATE == type ) {

            Date date = feature.getDate(attr.getIndex());

            fieldString = formatter.getFieldString(date);

            safeEncode(fieldString, 8, false);

        } else if( DataTypes.DOUBLE == type ) {

            double d = feature.getDouble(attr.getIndex());

            fieldString  = formatter.getFieldString(

                fieldLen, header.getFieldDecimalCount(dbfFieldIndex), d

            );

            safeEncode(fieldString, fieldLen, false);

        } else if( DataTypes.FLOAT == type ) {

            float f = feature.getFloat(attr.getIndex());

            fieldString = formatter.getFieldString(

                fieldLen, header.getFieldDecimalCount(dbfFieldIndex), f

            );

            safeEncode(fieldString, fieldLen, false);

        } else if( DataTypes.INT == type ) {

            int integer = feature.getInt(attr.getIndex());

            fieldString = formatter.getFieldString(

                fieldLen, header.getFieldDecimalCount(dbfFieldIndex), integer

            );

            safeEncode(fieldString, fieldLen, false);

        } else if( DataTypes.LONG == type ) {

            long l = feature.getLong(attr.getIndex());

            fieldString = formatter.getFieldString(

                fieldLen, header.getFieldDecimalCount(dbfFieldIndex),l

            );

            safeEncode(fieldString, fieldLen, false);

        } else if( DataTypes.STRING == type ) {

            String s = feature.getString(attr.getIndex());

            safeEncode(StringUtils.defaultIfEmpty(s, ""), fieldLen, true);

        } else {

            // Si no conocemos el tipo intentamos guardarlo como un string

            String s = feature.getString(attr.getIndex());

            safeEncode(StringUtils.defaultIfEmpty(s, ""), fieldLen, true);

        }

    }

    /**

     * Returns a safely padded (and potentially truncated) string 

     * 

     * This may truncate some record, but it is required to ensure

     * that the field limit is not overflowed when using 

     * variable-length charsets such as UTF-8.

     * @throws UnsupportedEncodingException 

     */

    private void safeEncode(String in, int limit, boolean rightPadding) throws UnsupportedEncodingException {

            try {

                    byte[] encodedString = in.getBytes(this.charset);

                    if (encodedString.length>limit) {

                            // too long, truncating

                            /*

                             * The block code bellow is equivalent to this simple code

                             * fragment:

                    if (rightPadding) {

                            in = in.substring(0, in.length()-1);

                            encodedString = in.getBytes(charset);

                    }

                    else {

                            in.substring(1, in.length());

                            encodedString = in.getBytes(charset);

                    }

                    However, the implemented algorithm has a much better performance

                    for the average and worst cases (when the input string has a lot

                    of multibyte characters), while keeping a good performance

                    for the best case (when all the characters in the input string

                    can be represented as single bytes using the selected charset).

                    The general strategy is to compute the deviation from the

                    required maximum number of bytes (limit) and the actual number

                    of bytes of the encoded String.

                    Then, we use this deviation to estimate the amount of characters

                    to truncate, based on the average factor of bytes per char in the

                    input string.

                    We truncate the string using this approach until the deviation

                    gets stable.

                    Finally, as we should be close enough to the right truncation position,

                    we increment/decrement the truncated string by only 1 character, to

                    ensure we truncate in the exact position. 

                             */

                            String str = in;

                            int estimatedDiff, deviation;

                            int deviationPrev;

                            double ratio;

                            byte[] encodedChar;

                            int truncatePos = 0;

                            deviation = encodedString.length - limit;

                            deviationPrev = deviation - 1;

                            while(Math.abs(deviation)>Math.abs(deviationPrev) && str.length()>0) {

                                    ratio = ((double)encodedString.length) / ((double)str.length());

                                    // apply the estimated diff, ensuring it is at least >= 1.0 in absolute value

                                    estimatedDiff = Math.max((int)(((double)deviation)/ratio), (int)(Math.signum(deviation)*1));

                                    // too long, truncating

                                    if (rightPadding) {

                                            truncatePos = Math.max(str.length()-estimatedDiff, 0);

                                            str = in.substring(0, truncatePos);

                                    }

                                    else {

                                            truncatePos = Math.max(truncatePos + estimatedDiff, 0);

                                            str = in.substring(truncatePos);                                  

                                    }

                                    encodedString = str.getBytes(charset);

                                    deviationPrev = deviation;

                                    deviation = encodedString.length - limit;

                            }

                            // now we are close enough, get the exact position for truncating

                            while (encodedString.length>limit) {

                                    // too long, truncating

                                    //                                      System.out.println("truncating");

                                    if (rightPadding) {

                                            str = in.substring(0, str.length()-1);

                                    }

                                    else {

                                            truncatePos = truncatePos + 1;

                                            str = in.substring(truncatePos);

                                    }

                                    encodedString = str.getBytes(charset);

                            }

                            while (encodedString.length<limit && str.length()<in.length()) {

                                    // Extend if necessary:

                                    // 1 - Get the length in bytes of the next char

                                    // 2 - Add the char to the substring if we are still within the limits 

                                    //                                      System.out.println("extending");

                                    if (rightPadding) {

                                            encodedChar = in.substring(str.length(), str.length()+1).getBytes(charset);

                                    }

                                    else {

                                            encodedChar = in.substring(truncatePos-1, truncatePos).getBytes(charset);

                                            //                                              System.out.println(encodedChar);

                                            //                                              System.out.println(encodedChar.length);

                                            //                                              System.out.println(testStrings[i].substring(truncatePos-1, truncatePos));

                                    }

                                    //                                      System.out.println(testStrings[i].substring(in.length(), in.length()+1));

                                    if ((encodedString.length + encodedChar.length)>limit) {

                                            // one more char would overflow the limit

                                            break;

                                    }

                                    // too short, extending

                                    if (rightPadding) {

                                            str = in.substring(0, str.length()+1);

                                    }

                                    else {

                                            truncatePos = truncatePos - 1;

                                            str = in.substring(truncatePos);

                                    }

                                    encodedString = str.getBytes(charset);

                            }

                    }

                    if (rightPadding) {

                            buffer.put(encodedString);

                    }

                    if (encodedString.length<limit) {

                            // too short, padding

                            int i = encodedString.length;

                            while (i<limit) {

                                    blank.position(0);

                                    buffer.put(blank);

                                    i=i+blankSize;

                            }

                            if (i>limit) {

                                    // Might happen for instance if charset is UTF16 and the

                                    // limit of characters in the field is an odd number

                                    throw new UnsupportedEncodingException(new Exception("Impossible to encode this DBF using the selected charset"));

                            }

                    }

                    if (!rightPadding) {

                            buffer.put(encodedString);

                    }

            }

                catch(BufferOverflowException exc) {

                        // Might happen for instance if charset is UTF16 and the

                        // limit of characters in the field is an odd number

                        throw new UnsupportedEncodingException(exc);

                }

    }

    /**

     * Returns a safely padded (and potentially truncated) string 

     * 

     * This may truncate some record, but it is required to ensure

     * that the field limit is not overflowed when using 

     * variable-length charsets such as UTF-8.

     * 

     * This implementation is not used but it is kept here for reference.

     * It is fully equivalent to the {@link #safeEncode(String, int, boolean)}

     * method and easier to understand, but this implementation is much

     * slower for any multibyte charset (such as UTF-8).

     *  

     * @throws UnsupportedEncodingException 

     */

    private void safeEncodeSlow(String in, int limit, boolean rightPadding) throws UnsupportedEncodingException {

            try {

                    byte[] encodedString = in.getBytes(this.charset);

                    while (encodedString.length>limit) {

                            // too long, truncating

                            if (rightPadding) {

                                    in = in.substring(0, in.length()-1);

                                    encodedString = in.getBytes(charset);

                            }

                            else {

                                    in.substring(1, in.length());

                                    encodedString = in.getBytes(charset);

                            }

                    }

                    if (rightPadding) {

                            buffer.put(encodedString);

                    }

                    if (encodedString.length<limit) {

                            // too short, padding

                            int i = encodedString.length;

                            while (i<limit) {

                                    blank.position(0);

                                    buffer.put(blank);

                                    i=i+blankSize;

                            }

                            if (i>limit) {

                                    throw new UnsupportedEncodingException(new Exception("Impossible to encode this DBF using the selected charset"));

                            }

                    }

                    if (!rightPadding) {

                            buffer.put(encodedString);

                    }

            }

            catch(BufferOverflowException exc) {

                    // Might happen for instance if charset is UTF16 and the

                    // limit of characters in the field is an odd number

                    throw new UnsupportedEncodingException(exc);

            }

    }

    /**

     * Release resources associated with this writer. <B>Highly recommended</B>

     * 

     * @throws CloseException

     * @throws IOException

     *             If errors occur.

     */

    public void close() throws CloseException {

        // IANS - GEOT 193, bogus 0x00 written. According to dbf spec, optional

        // eof 0x1a marker is, well, optional. Since the original code wrote a

        // 0x00 (which is wrong anyway) lets just do away with this :)

        // - produced dbf works in OpenOffice and ArcExplorer java, so it must

        // be okay.

        // buffer.position(0);

        // buffer.put((byte) 0).position(0).limit(1);

        // write();

        if (headDrity) {

            try {

                this.writeHeader();

            } catch (WriteException e) {

                throw new CloseException("DbaseFileWriter", e);

            }

        }

        try {

            channel.close();

        } catch (IOException e) {

            throw new CloseException("DBF Writer", e);

        }

        if (buffer instanceof MappedByteBuffer) {

            // NIOUtilities.clean(buffer);

        }

        buffer = null;

        channel = null;

        formatter = null;

    }

    /** Utility for formatting Dbase fields. */

    public static class FieldFormatter {

        private StringBuffer buffer = new StringBuffer(255);

        private NumberFormat numFormat = NumberFormat

            .getNumberInstance(Locale.US);

        private Calendar calendar = Calendar.getInstance(Locale.US);

        private String emtpyString;

        private static final int MAXCHARS = 255;

        public FieldFormatter() {

            // Avoid grouping on number format

            numFormat.setGroupingUsed(false);

            // build a 255 white spaces string

            StringBuffer sb = new StringBuffer(MAXCHARS);

            sb.setLength(MAXCHARS);

            for (int i = 0; i < MAXCHARS; i++) {

                sb.setCharAt(i, ' ');

            }

            emtpyString = sb.toString();

        }

        public String getFieldString(int size, String s) {

            buffer.replace(0, size, emtpyString);

            buffer.setLength(size);

            if (s != null) {

                buffer.replace(0, size, s);

                if (s.length() <= size) {

                    for (int i = s.length(); i < size; i++) {

                        buffer.append(' ');

                    }

                }

            }

            buffer.setLength(size);

            return buffer.toString();

        }

        public String getFieldString(Date d) {

            if (d != null) {

                buffer.delete(0, buffer.length());

                calendar.setTime(d);

                int year = calendar.get(Calendar.YEAR);

                int month = calendar.get(Calendar.MONTH) + 1; // returns 0 based

                                                              // month?

                int day = calendar.get(Calendar.DAY_OF_MONTH);

                if (year < 1000) {

                    if (year >= 100) {

                        buffer.append("0");

                    } else

                        if (year >= 10) {

                            buffer.append("00");

                        } else {

                            buffer.append("000");

                        }

                }

                buffer.append(year);

                if (month < 10) {

                    buffer.append("0");

                }

                buffer.append(month);

                if (day < 10) {

                    buffer.append("0");

                }

                buffer.append(day);

            } else {

                buffer.setLength(8);

                buffer.replace(0, 8, emtpyString);

            }

            buffer.setLength(8);

            return buffer.toString();

        }

        public String getFieldString(int size, int decimalPlaces, double n) {

            buffer.delete(0, buffer.length());

            numFormat.setMaximumFractionDigits(decimalPlaces);

            numFormat.setMinimumFractionDigits(decimalPlaces);

            numFormat.format(n, buffer, new FieldPosition(

                NumberFormat.INTEGER_FIELD));

            return buffer.toString();

        }

    }

    public void setCharset(Charset charset) {

        this.charset = charset;

            blank = charset.encode(" ");

            blankSize = blank.limit();

    }

}