moving files over the network

[Alan Shiers]

Hi there,

I have this one recurring problem with a worker-thread on a client app that
I use to upload files to a server app. The problem seems to only exist when
dealing with compressed files ( *.zip ) and the like. After an upload of a
compressed file, I go to the server and try to open it with a compression
utility. Every time it shoots up a message saying that the file is not a
valid archive file. For the life of me I don't understand why my code works
with any other file type except for compressed files. I need an expert to
look over my code and explain why it fails when dealing with compressed file
types. I need to get this problem rectified and make my code more robust
and failsafe.

To just give a brief explanation of my approach to creating these methods,
which you'll see below, is to determine an appropriate buffer size in
accordance with the largest file of any group of files being uploaded.
Next, I iterate over an array of File objects using a for loop structure
wherein its body I call a method (uploadFile(int bufferSize, File file))
that handles each File seperately. In uploadFile(...) I create instances of
BufferedOutputStream and BufferedInputStream to read in a file and send it
out the wire. If the file is substantially large (bigger than 4kb), which
many zipped files are, then I send the data over the wire in 256 byte
chunks.

The method receiveMultipleFiles(), on the server-side, handles the incoming
data for this process.

Maybe I need to implement some kind of byte counting scheme to ensure the
exact number of bytes get read in and sent out the wire on the client side,
and the same thing on the server side? I don't know! I'm at a loss at this
point.

Please advise,

Alan Shiers

**************************************

//client side app worker-thread methods for uploading files
 
private void SendMultipleFiles()
  {
     int bufferSize = 0;
     rq.ref.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR));
     try
     {
       /*Set the buffer according to the largest filesize to 4Kb.
         The following buffer settings have been tested:
         For small files less than 2Mb a buffer of 4Kb seems to be adequate.
         However, for files between 2Mb - 30Mb a buffer of 50Kb is required.
         Files greater than 30Mb we'll set the buffer for 100Kb.
       */
       int largestFile = getLargestFileSize();
       if(largestFile <= 2097152) //less than or equal to 2Mb
       {
         bufferSize = 4096;
       }
       else if(largestFile > 2097152 && largestFile <= 31457190)
       {
         bufferSize = 51200;
       }
       else
       {
          bufferSize = 102400;
       }
       //Iterate through each element of the "files" array
       for(int i = 0; i < files.length; i++)
       {
          //Give the server time to prepare for the next file.
          Thread.sleep(1000);
          uploadFile(bufferSize, files[i]);
       }
 
       //System.out.println("Closed stream");
       rq.output.flush();
       rq.enableMenus();
       rq.ref.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
       //Get all the names of the files.
       String Names = "";
       for(int i = 0; i < files.length; i++)
       {
         Names = Names + files[i].getName() + "\n";
       }
       JOptionPane.showMessageDialog(rq.ref, "The following files were\n" +
                                             "uploaded:\n\n" + Names,
                                             "Upload Status:",
 
JOptionPane.INFORMATION_MESSAGE);
       rq.proceed = false; //reset
       timeElapsed = 0; //reset
    }
    catch(FileNotFoundException fnfe)
    {
      rq.ref.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
    }
    catch(IOException ioe)
    {
      rq.ref.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
    }
    catch(Exception e)
    {
      rq.ref.setCursor(Cursor.getPredefinedCursor(Cursor.DEFAULT_CURSOR));
    }
 
  }
  
  private void uploadFile(int BufferSize, File myFile)
  {
    try
    {
      bos = new BufferedOutputStream(rq.ref.connection1.getOutputStream(), BufferSize);
      bis = new BufferedInputStream(new FileInputStream(myFile), BufferSize);
 
      if((int)myFile.length() <= 4096)//less than or equal to 4Kb (initial buffer size)
      {
         byte[] data = new byte[(int)myFile.length()];
         bis.read(data);
         bos.flush();
         bos.write(data);
         bos.flush();
      }
      else
      { 
         //Set up the progresss bar
         final int FileSize = (int)myFile.length();
         System.out.println("FileSize = " + FileSize);
         rq.ref.jProgressBar1.setMinimum(0);
         rq.ref.jProgressBar1.setMaximum(FileSize);
         rq.ref.jProgressBar1.setStringPainted(true);
         rq.ref.jProgressBar1.setVisible(true);
         rq.ref.textLabel.setText("Uploading: " + myFile.getName());
         //set a default size for the byte array
         byte[] data = new byte[256];
 
         /*
 
         //Figure out how many whole 256 chunks can be had from this file.
         int Sum = ((int)myFile.length() / 256);
         for(int j = 0; j < Sum; j++)
         {
            bis.read(data);
            bos.write(data);
            bos.flush();
            final int count = (256*j);
            SwingUtilities.invokeLater(new Runnable()
            {
               public void run()
               {
    rq.ref.jProgressBar1.setValue(count);
               }
            });
            yield();
         }
         //Write the remaining bytes to the stream
         //which would be less than 256 bytes
         bis.read(data);
         bos.write(data);
         bos.flush();
         //Finish the progressBar
         SwingUtilities.invokeLater(new Runnable()
         {
           public void run()
           {
              rq.ref.jProgressBar1.setValue(FileSize);
           }
         });
         yield();
         rq.ref.jProgressBar1.setValue(0);
       }
 
       */
 
       //int size = 0;
       int cnt = 0;
       for(; :wink: 
       {
         cnt = bis.read(data);
         if(cnt <= 0)break;
         bos.write(data,0,cnt);
         bos.flush();
         //size+=cnt;
         final int count = (size);
            SwingUtilities.invokeLater(new Runnable()
            {
               public void run()
               {
    rq.ref.jProgressBar1.setValue(count);
               }
            });
            yield();
       }
       //System.out.println("size = " + size);
                    
       //Close the stream to the file
       if(bis != null)
          bis.close();
       rq.ref.jProgressBar1.setVisible(false);
     }

**********************************************

//server-side app that recieves the incoming files
private void receiveMultipleFiles()
  {
    //Check first to see if there is enough space for all these files
    boolean enoughSpace = isEnoughSpaceAvailable(FileSize);
    //System.out.println("enoughSpace = " + enoughSpace);
    if (enoughSpace == false)
    {
      MultipleFileCount = 0; //reset
      MultipleFileNames = null; //reset
      MultipleFileSizes = null; //reset
      return;
    }
    File myFile = null;
    String separator = "";
 
    if(getPlatform() == session.WINDOWS)
      separator = "\\" + "\\";
    else
      separator = "/";
 
    int bufferSize = 0;
    /*Set the buffer according to the largest filesize to 4Kb.
      The following buffer settings have been tested:
      For small files less than 2Mb a buffer of 4Kb seems to be adequate.
      However, for files between 2Mb - 30Mb a buffer of 50Kb is required.
      Files greater than 30Mb we'll set the buffer for 100Kb.
    */
     int largestFile = getLargestFileSize();
     if(largestFile <= 2097152) //less than or equal to 2Mb
     { //System.out.println("Largest File < 2Mb: " + largestFile);
       bufferSize = 4096;
     }
     else if(largestFile > 2097152 && largestFile <= 31457190)
     { //System.out.println("Largest File > 2Mb & < 30Mb: " + largestFile);
       bufferSize = 51200;
     }
     else
     { //System.out.println("Largest File > 30Mb: " + largestFile);
       bufferSize = 102400;
     }
     try
     {
       //iterate through all the incoming files
       int i;
       for(i = 0; i < MultipleFileCount; i++)
       {
         //get the path and name of each file
         myFile = new File(path + separator + MultipleFileNames[i]);
         myFile.createNewFile(); //create the file
         if(myFile.exists())
         {
            downloadFile(bufferSize, myFile, i);
         }
 
       }//End for loop
       output.writeInt(session.UPLOAD_SUCCESSFUL);
       //System.out.println("Sent Upload Successful");
       output.flush();
   }
 
   private void downloadFile(int BufferSize, File myFile, int MultipleFileSizeIndex)
 {
   BufferedInputStream in = null;
   BufferedOutputStream out = null;
      
   //Set the buffer for the InputStream and OutputStream  
   try
   {
     in = new BufferedInputStream(connection1.getInputStream(), BufferSize);
     out = new BufferedOutputStream(new FileOutputStream(myFile), BufferSize);
   }
   catch(FileNotFoundException fnfe)
   {
     try
     {
        output.writeInt(session.FILE_NOT_FOUND_EXCEPTION);
        //Close the stream to the file.
        if(out != null)
           out.close();
        output.flush();
      }
      catch(IOException ioe){ErrLogger.log(Level.SEVERE, this.getClass().getName(), "downloadFile", "Caught Exception", ioe);}
      MultipleFileCount = 0; //reset
      MultipleFileNames = null; //reset
      MultipleFileSizes = null; //reset
      return;
   }
   catch(IOException io)
   {
      try
     {
        //Close the stream to the file.
        if(out != null)
           out.close();
        output.flush();
      }
      catch(IOException ioe){ErrLogger.log(Level.SEVERE, this.getClass().getName(), "downloadFile", "Caught Exception", ioe);}
      MultipleFileCount = 0; //reset
      MultipleFileNames = null; //reset
      MultipleFileSizes = null; //reset
      return;
   }
   try
   {
     if((int)MultipleFileSizes[MultipleFileSizeIndex] <= 4096)//less than or equal 4Kb 
     {
       byte[] data = new byte[(int)MultipleFileSizes[MultipleFileSizeIndex]];
       in.read(data);
       out.flush();
       out.write(data);
       out.flush();
      }
      else
      {
         byte[] data = new byte[256];
         //System.out.println("FileSize = " + (int)MultipleFileSizes[MultipleFileSizeIndex]);
         //Figure out how many whole 256 chunks can be had from this file.
         int Sum = ((int)MultipleFileSizes[MultipleFileSizeIndex] / 256);
         for(int j = 0; j < Sum; j++)
         {
           in.read(data);
           out.write(data);
           out.flush();
         }
         //Figure out how many remaining bytes there are left
         Sum = ((int)MultipleFileSizes[MultipleFileSizeIndex] - (Sum * 256));
           
         //Write the remaining bytes to the file
         in.read(data,0,Sum);
         out.write(data,0,Sum);
         out.flush();
 
         //System.out.println("Finished download");         
         
      }
        
      //Close the stream to each file
      if(out != null)
         out.close();
   }
   catch(IOException ioe)
   {
       try
       {
         output.writeInt(session.UPLOAD_UNSUCCESSFUL);
         output.flush();         
       }
       catch(IOException io){ErrLogger.log(Level.SEVERE, this.getClass().getName(), "downloadFile", "Caught Exception", io);}
    }
    catch(Exception e)
    {
       try
       {
         output.writeInt(session.UPLOAD_UNSUCCESSFUL);
         output.flush();
       }
       catch(IOException io){ErrLogger.log(Level.SEVERE, this.getClass().getName(), "downloadFile", "Caught Exception", io);}
    }
   
 }
   catch(FileNotFoundException fnfe)
   {
      try
      {
        output.writeInt(session.FILE_NOT_FOUND_EXCEPTION);
        output.flush();
      }
      catch(IOException ioe){ErrLogger.log(Level.SEVERE,
this.getClass().getName(), "receiveMultipleFiles", "Caught Exception",
ioe);}
    }
    catch(SecurityException se)
    {
      if(DirAdmin.hasInterface)
        myServer.display("SecurityException occurred in method
receiveNonTextFile()");
      else
        ErrLogger.log(Level.SEVERE, this.getClass().getName(),
"receiveMultipleFiles", "Caught Exception", se);
      try
      {
        output.writeInt(session.UPLOAD_UNSUCCESSFUL_SECURITY_EXCEPTION);
        output.flush();
      }
      catch(IOException ioe){ErrLogger.log(Level.SEVERE,
this.getClass().getName(), "receiveMultipleFiles", "Caught Exception",
ioe);}
    }
    catch(IOException ioe)
    {
       try
       {
         output.writeInt(session.UPLOAD_UNSUCCESSFUL);
         output.flush();
       }
       catch(IOException io){ErrLogger.log(Level.SEVERE,
this.getClass().getName(), "receiveMultipleFiles", "Caught Exception", io);}
    }
    catch(Exception e)
    {
       try
       {
         output.writeInt(session.UPLOAD_UNSUCCESSFUL);
         output.flush();
       }
       catch(IOException io){ErrLogger.log(Level.SEVERE,
this.getClass().getName(), "receiveMultipleFiles", "Caught Exception", io);}
    }
    finally
    {
      MultipleFileCount = 0; //reset
      MultipleFileNames = null; //reset
      MultipleFileSizes = null; //reset
    }
 }
 

[Balasubramani Amirthalingam]

Hi

I am also facing the problem to upload files in weblogic.
I got following exception like
java.net.ProtocolException: EOF after reading only: '16384' of: '269635' promised bytes, out of which at least: '0' were already buffered.

I am uploading one file in server but my code is working fine if the file size is less than 4096 bytes, if i hv given more than that i got exception.
if u refer my code, u can get some idea, simmilarly if u get the solution for that it will be helpful for me..

my code is...
private void uploadFunction(HttpServletRequest req,HttpServletResponse res){


String contentType = req.getContentType();
System.out.println("Content type is :: " +contentType);
if ((contentType != null) && (contentType.indexOf("multipart/form-data") >= 0)) {
try{
DataInputStream in = new DataInputStream(req.getInputStream());
int formDataLength = req.getContentLength();

if(formDataLength<=5000000){

byte dataBytes[] = new byte[formDataLength];
int byteRead = 0;
int totalBytesRead = 0;

while (totalBytesRead < formDataLength) {
byteRead = in.read(dataBytes, totalBytesRead, formDataLength);
totalBytesRead += byteRead;
}
String file = new String(dataBytes);

String saveFile = file.substring(file.indexOf("filename=\"") + 10);

saveFile = saveFile.substring(0, saveFile.indexOf("\n"));
saveFile = saveFile.substring(saveFile.lastIndexOf("\\") + 1,saveFile.indexOf("\""));



int lastIndex = contentType.lastIndexOf("=");
String boundary = contentType.substring(lastIndex + 1,contentType.length());
int pos;
pos = file.indexOf("filename=\"");
pos = file.indexOf("\n", pos) + 1;
pos = file.indexOf("\n", pos) + 1;
pos = file.indexOf("\n", pos) + 1;
int boundaryLocation = file.indexOf(boundary, pos) - 4;
int startPos = ((file.substring(0, pos)).getBytes()).length;
int endPos = ((file.substring(0, boundaryLocation)).getBytes()).length;
if(saveFile.length()!=0){
saveFile="D:\\soft\\"+saveFile;
FileOutputStream fileOut = new FileOutputStream(saveFile);
fileOut.write(dataBytes, startPos, (endPos - startPos));
fileOut.flush();
fileOut.close();
}


}
}catch(IOException e){
System.out.println("except.........."+e);
e.printStackTrace();
}
}
thx in advance
Balu

[Alan Shiers]

Hi there,

Here is a program I came across in another newgroup on the net. I scoured many newsgroups for many hours and finally found this little gem. This solved my problems. Like they say, "KISS" Keep it simple stupid!

This program acts as both the server app and client app. So, you can run two instances of this class at the same time on one workstation. The main() method will start the program either as the server app or the client app depending on whether you pass an arguement to main() or not. The arguement can be anything.

Hope this helps you as it did me.

Alan

*********************************************

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.net.ServerSocket;
import java.net.Socket;
 
public class FileTransfer
{
  public static final int BUFFER_SIZE = 1024;
 
  private byte[] buffer;
 
  public FileTransfer()
  {
    buffer = new byte[BUFFER_SIZE];
  }
 
  public void startServer()throws Exception
  {
    ServerSocket socket = new ServerSocket(9000);
    Socket client = socket.accept();
    BufferedInputStream in = new BufferedInputStream(new FileInputStream("someFile.zip"));
    BufferedOutputStream out = new BufferedOutputStream(client.getOutputStream());
    int len = 0;
    while((len = in.read(buffer)) > 0)
    {
      out.write(buffer,0,len);
      System.out.print("#");
    }
    in.close();
    out.flush();
    out.close();
    client.close();
    socket.close();
    System.out.println("\nDone!");
  }
 
  public void startClient()throws Exception
  {
    Socket socket = new Socket("localhost",9000);
    BufferedInputStream in = new BufferedInputStream(socket.getInputStream());
    BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream("someFile.zip"));
    int len = 0;
    while((len=in.read(buffer))>0)
    {
      out.write(buffer,0,len);
      System.out.print("#");
    }
    in.close();
    out.flush();
    out.close();
    socket.close();
    System.out.println("\nDone!");
  }
 
  public static void main(String[] args)throws Exception
  {
    FileTransfer test = new FileTransfer();
    if(args.length == 0)
      test.startServer();
    else
      test.startClient();
  } 
}