problem base64 encoding the 3 hex bytes E2 80 A9

[Miguel Capo]

Hi there,
I am having problems when I base64 encode a file that contains the 3 hex bytes E2 80 A9. When I decode the file I get the hex byte 3F.

Can some shed some light?

Thanks

[Jesper de Jong]

Show us your code. What are you using to do the Base64 encoding and decoding?

(This question isn't really an Advanced Java question).

[Miguel Capo]

Thanks for your help,

The classes below are the ones that I am using to do the encoding and the decoding.

 
import java.util.Arrays;
 
/**
 * Utility class to do Base64 encoding, as defined by RFC 2045,
 * section 6.8 (http://www.ietf.org/rfc/rfc2045.txt)
 * Uses the same class and function names as Sun's implementation from
 * sun.misc
 */
public class BASE64Encoder {
 
  /**
   * Convert a byte to an integer.  Needed because in Java bytes
   * are signed, and for Base64 purposes they are not.  If not done
   * this way, when converted to an int, 0xFF will become -127
   * @param b Byte value to be converted
   * @return Value as an integer, as if byte was unsigned
   */
  private int uByteToInt(byte b) {
    if (b >= 0) {
      return (int) b;
    }
 
    return 256 + b;
  }
 
  /**
   * Encode an array of bytes using Base64
   * @param data[] The bytes to be encoded
   * @return A valid Base64 representation of the input
   */
  public String encode(byte data[]) {
    // Base64 encoding yields a String that is 33% longer than the byte array
    int charCount = ((data.length * 4) / 3) + 4;
 
    // New lines will also be needed for every 76 charactesr, so allocate a
    // StringBuffer that is long enough to hold the full result without
    // having to expand later
    StringBuffer result = new StringBuffer(
        (charCount * 77) / 76);
 
    int byteArrayLength = data.length;
    int byteArrayIndex = 0;
    int byteTriplet = 0;
    while (byteArrayIndex < byteArrayLength - 2) {
      byteArrayIndex = processData(
          data, byteArrayIndex, result);
 
    }
 
    byteArrayIndex = checkIfWeHave1ByteLeftOver(
        byteArrayIndex, byteArrayLength, data, result);
    checkIfWeHaveTwoBytesLeftOver(
        byteArrayIndex, byteArrayLength, data, result);
 
    return result.toString();
  }
 
  private int processData(
      byte[] data, int index,
      StringBuffer result) {
    int byteTriplet;
    // Build the 24 bit byte triplet from the input data
    byteTriplet = uByteToInt(data[index++]);
    // Each input byte contributes 8 bits to the triplet
    byteTriplet <<= 8;
    byteTriplet |= uByteToInt(data[index++]);
    byteTriplet <<= 8;
    byteTriplet |= uByteToInt(data[index++]);
 
    // Look at the lowest order six bits and remember them
    byte b4 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
    // Move the byte triplet to get the next 6 bit value
    byteTriplet >>= 6;
    byte b3 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
    byteTriplet >>= 6;
    byte b2 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
    byteTriplet >>= 6;
    byte b1 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
 
    // Add the Base64 encoded character to the result String
    result.append(byteToChar(b1));
    result.append(byteToChar(b2));
    result.append(byteToChar(b3));
    result.append(byteToChar(b4));
 
    // There are 57 bytes for every 76 characters, so wrap the line when needed
    if (index % 57 == 0) {
      result.append("\n");
    }
    return index;
  }
 
  private void checkIfWeHaveTwoBytesLeftOver(
      int index,
      int length,
      byte[] byteArray,
      StringBuffer sb) {
    int byteTriplet;
    if (index == length - 2) {
      // Convert our two bytes to an int
      byteTriplet = twoBytesToInt(byteArray, index);
      // Right pad the third 6 bit value with zeros
      byteTriplet <<= 2;
 
      byte b3 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
      byteTriplet >>= 6;
      byte b2 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
      byteTriplet >>= 6;
      byte b1 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
 
      sb.append(byteToChar(b1));
      sb.append(byteToChar(b2));
      sb.append(byteToChar(b3));
 
      // Add "==" to the output to make it a multiple of 4 Base64 characters
      sb.append("=");
    }
  }
 
  private int checkIfWeHave1ByteLeftOver(
      int index,
      int length,
      byte[] byteArray,
      StringBuffer sb) {
    int byteTriplet;
    if (index == length - 1) {
      // Convert our one byte to an int
      byteTriplet = uByteToInt(byteArray[index++]);
      // Right pad the second 6 bit value with zeros
      byteTriplet <<= 4;
 
      byte b2 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
      byteTriplet >>= 6;
      byte b1 = (byte) (Base64Constants.SIX_BIT_MASK & byteTriplet);
 
      sb.append(byteToChar(b1));
      sb.append(byteToChar(b2));
 
      // Add "==" to the output to make it a multiple of 4 Base64 characters
      sb.append("==");
    }
    return index;
  }
 
  private int twoBytesToInt(byte[] data, int byteArrayIndex) {
    int byteTriplet;
    byteTriplet =
        uByteToInt(data[byteArrayIndex++]);
    byteTriplet <<= 8;
    byteTriplet |= uByteToInt(data[byteArrayIndex++]);
    return byteTriplet;
  }
 
  /**
   * Convert a byte between 0 and 63 to its Base64 character equivalent
   * @param b Byte value to be converted
   * @return Base64 char value
   */
  private char byteToChar(byte b) {
    if (b < Base64Constants.LOWER_CASE_A_VALUE) {
      return (char) ('A' + b);
    }
 
    if (b < Base64Constants.ZERO_VALUE) {
      return (char) ('a' + (b - Base64Constants.LOWER_CASE_A_VALUE));
    }
 
    if (b < Base64Constants.PLUS_VALUE) {
      return (char) ('0' + (b - Base64Constants.ZERO_VALUE));
    }
 
    if (b == Base64Constants.PLUS_VALUE) {
      return '+';
    }
 
    if (b == Base64Constants.SLASH_VALUE) {
      return '/';
    }
 
    throw
        new IllegalArgumentException(
            "Byte " +
            new Integer(b) +
            " is not a valid Base64 value");
  }
 
  /**
   * Simple test method to make sure everything works correctly
   * Creates 100 randomly sized arrays of random bytes, encodes them,
   * decodes them, and checks to make sure the result matches the input
   */
  public static void main(String args[]) throws Exception {
    testCodec();
 
  }
 
  private static void testCodec() {
//      sun.misc.BASE64Encoder encoder = new sun.misc.BASE64Encoder();
//      sun.misc.BASE64Decoder decoder = new sun.misc.BASE64Decoder();
 
    for (int j = 0; j < 100; j++)
      test(new BASE64Encoder(), new BASE64Decoder());
 
  }
 
  private static void test(BASE64Encoder encoder, BASE64Decoder decoder) {
    byte test[] = new byte[(int) (100000 * Math.random())];
    for (int i = 0; i < test.length; i++) {
      test[i] = (byte) (256 * Math.random());
    }
 
    String string = encoder.encode(test);
    byte result[] = decoder.decodeBuffer(string);
 
    if (!Arrays.equals(test, result) || test.length != result.length) {
      System.out.println("ARRAYS DO NOT MATCH!");
    }
  }
}
 
 
/**
 * Utility class to do Base64 decoding, as defined by RFC 2045,
 * section 6.8 (http://www.ietf.org/rfc/rfc2045.txt)
 * Uses the same class and function names as Sun's implementation from
 * sun.misc
 */
public class BASE64Decoder {
 
  /**
   * Bit mask for one byte worth of bits in Base64 encoding.
   * Equivalent to binary value 11111111b.
   */
  private static final int EIGHT_BIT_MASK = 0xFF;
 
  /**
   * Decode an input String using Base64
   * @param data The String to be decoded
   * @return The appropriate byte array
   */
  public byte[] decodeBuffer(String data) {
    // Create a wrapper around the input to screen out non-Base64 characters
    StringWrapper wrapper = new StringWrapper(data);
    // A Base64 byte array is 75% the size of its String representation
    int length = wrapper.getUsefulLength() * 3 / 4;
 
    byte byteArray[] = new byte[length];
 
    int byteTriplet = 0;
    int index = 0;
 
    // Continue until we have less than 4 full characters left to
    // decode in the input.
    while (index + 2 < length) {
 
      index = processByteArray(
          wrapper,
          byteArray,
          index);
 
    }
 
    checkIfWeHave1ByteLeft(
        index,
        length,
        wrapper, byteArray);
 
    checkIfWeHave2BytesLeft(index,
                            length,
                            wrapper,
                            byteArray);
 
    return byteArray;
  }
 
  private void checkIfWeHave2BytesLeft(int index, int length, StringWrapper wrapper, byte[] byteArray) {
    int byteTriplet;
    if (index == length - 2) {
      // Take out the last three characters from the String
      byteTriplet = charToInt(wrapper.getNextUsefulChar());
      byteTriplet <<= 6;
      byteTriplet |= charToInt(wrapper.getNextUsefulChar());
      byteTriplet <<= 6;
      byteTriplet |= charToInt(wrapper.getNextUsefulChar());
 
      // Remove the padded zeros
      byteTriplet >>= 2;
      byteArray[index + 1] = (byte) (byteTriplet & EIGHT_BIT_MASK);
      byteTriplet >>= 8;
      byteArray[index] = (byte) (byteTriplet & EIGHT_BIT_MASK);
    }
  }
 
  private void checkIfWeHave1ByteLeft(int byteIndex, int byteArrayLength, StringWrapper wrapper, byte[] result) {
    int byteTriplet;
    if (byteIndex == byteArrayLength - 1) {
      // Take out the last two characters from the String
      byteTriplet = charToInt(wrapper.getNextUsefulChar());
      byteTriplet <<= 6;
      byteTriplet |= charToInt(wrapper.getNextUsefulChar());
 
      // Remove the padded zeros
      byteTriplet >>= 4;
      result[byteIndex] = (byte) (byteTriplet & EIGHT_BIT_MASK);
    }
  }
 
  private int processByteArray(StringWrapper wrapper, byte[] result, int byteIndex) {
    int byteTriplet;
    // Package a set of four characters into a byte triplet
    // Each character contributes 6 bits of useful information
    byteTriplet = charToInt(wrapper.getNextUsefulChar());
    byteTriplet <<= 6;
    byteTriplet |= charToInt(wrapper.getNextUsefulChar());
    byteTriplet <<= 6;
    byteTriplet |= charToInt(wrapper.getNextUsefulChar());
    byteTriplet <<= 6;
    byteTriplet |= charToInt(wrapper.getNextUsefulChar());
 
    // Grab a normal byte (eight bits) out of the byte triplet
    // and put it in the byte array
    result[byteIndex + 2] = (byte) (byteTriplet & EIGHT_BIT_MASK);
    byteTriplet >>= 8;
    result[byteIndex + 1] = (byte) (byteTriplet & EIGHT_BIT_MASK);
    byteTriplet >>= 8;
    result[byteIndex] = (byte) (byteTriplet & EIGHT_BIT_MASK);
    byteIndex += 3;
    return byteIndex;
  }
 
  /**
   * Convert a Base64 character to its 6 bit value as defined by the mapping.
   * @param c Base64 character to decode
   * @return int representation of 6 bit value
   */
  private int charToInt(char c) {
    if (c >= 'A' && c <= 'Z') {
      return c - 'A';
    }
 
    if (c >= 'a' && c <= 'z') {
      return (c - 'a') + Base64Constants.LOWER_CASE_A_VALUE;
    }
 
    if (c >= '0' && c <= '9') {
      return (c - '0') + Base64Constants.ZERO_VALUE;
    }
 
    if (c == '+') {
      return Base64Constants.PLUS_VALUE;
    }
 
    if (c == '/') {
      return Base64Constants.SLASH_VALUE;
    }
 
    throw new IllegalArgumentException(c + " is not a valid Base64 character.");
  }
 
  /**
   * Simple class to wrap around the String input to ignore all of the
   * non-Base64 characters in the input.  Note that although '=' is
   * a valid character, it does not contribute to the total number
   * of output bytes, and is therefore ignored
   */
  private class StringWrapper {
 
    /**
     * The input String to be decoded
     */
    private String mString;
 
    /**
     * Current position in the String
     */
    private int mIndex = 0;
 
    /**
     * Total number of Base64 characters in the input
     */
    private int mUsefulLength;
 
    /**
     * @param c Character to be examined
     * @return Whether or not the character is a Base64 character
     */
    private boolean isUsefulChar(char c) {
      return (c >= 'A' && c <= 'Z') ||
          (c >= 'a' && c <= 'z') ||
          (c >= '0' && c <= '9') ||
          (c == '+') ||
          (c == '/');
    }
 
    /**
     * Create the wrapper and determine the number of Base64 characters in
     * the input
     * @param s Input String to be decoded
     */
    public StringWrapper(String s) {
      mString = s;
      mUsefulLength = 0;
      int length = mString.length();
      for (int i = 0; i < length; i++) {
        if (isUsefulChar(mString.charAt(i))) {
          mUsefulLength++;
        }
      }
    }
 
    /**
     * @return Total number of Base64 characters in the input.  Does
     * not include '='
     */
    public int getUsefulLength() {
      return mUsefulLength;
    }
 
    /**
     * Traverse the String until hitting the next Base64 character.
     * Assumes that there is still another valid Base64 character
     * left in the String.
     */
    public char getNextUsefulChar() {
      char result = '_';  // Start with a non-Base64 character
      while (!isUsefulChar(result)) {
        result = mString.charAt(mIndex++);
      }
 
      return result;
    }
  }
}
 
public class Base64Constants {
  /**
   * Byte value that maps to 'a' in Base64 encoding
   */
  final static int LOWER_CASE_A_VALUE = 26;
  /**
   * Byte value that maps to '0' in Base64 encoding
   */
  final static int ZERO_VALUE = 52;
  /**
   * Byte value that maps to '+' in Base64 encoding
   */
  final static int PLUS_VALUE = 62;
  /**
   * Byte value that maps to '/' in Base64 encoding
   */
  final static int SLASH_VALUE = 63;
  /**
   * Bit mask for one character worth of bits in Base64 encoding.
   * Equivalent to binary value 111111b.
   */
  final static int SIX_BIT_MASK = 63;

 
}

[ May 11, 2007: Message edited by: Jim Yingst ]

[Jesper de Jong]

So you wrote your own Base64 encoder and decoder?

Is there a special reason why you wrote this yourself; why don't you use a well-tested library like Apache Jakarta Commons Codec?

If you encode and decode something and the result is different from the input, then there's obviously a bug somewhere in your code. Use a debugger to step through your code and see if it is doing what you expect it to do. You could also put System.out.println(...); statements in your code to print the value of variables at strategic places to see if the program is doing what you expect it to do.

[Miguel Capo]

Thanks for your advice,
I am using the Apache Jakarta Commons Codec and the problem has disappeared.