Splitting up messages in a text file

[Rich Charlton]

Hi,

Im working on a basic email filter, which will read in a text file of several hundred email messages, increment a counter for an individual message, tokenise it, count number of occurences of token in that email then store in a dictionary. I have built a couple of classes that do certain things and they work ok. Although i cant figure out how i can split up each individual message in the text file. At the moment i just read the whole text file in, with no method of splitting the text file up into individual messages. The messages in the text file are split by 10 dashes (----------). Here are my classes so far:
Tokeniser.java

import java.io.*;
import java.util.*;
import javax.swing.JOptionPane;
import java.util.ArrayList;
 
public class Tokeniser {
 
public HashSet unCommonHS = new HashSet();
public int numberOfTokens = 0;
 
public Tokeniser() 
{
String fileName = "SPAM.txt";
String aLine;
ComWords cm = new ComWords();
HashSet cwSet = cm.getCommonWordsHS();
 
if (cm==null) return;
try 
{
BufferedReader bR = new BufferedReader(new FileReader(fileName));
 
while((aLine=bR.readLine())!= null) 
{
checkForCommonWord (aLine, cwSet, unCommonHS);
}
bR.close();
 
Object[] ucw = unCommonHS.toArray();
 
for (int i=0; i<ucw.length; i++) {    
numberOfTokens++;
System.out.print(ucw[i]+" ");
if (i!=0 && (i%1)==0) 
System.out.println();
}
Set sm = new TreeSet(unCommonHS);      
for (Iterator it = sm.iterator(); it.hasNext(); ) 
        {    
            numberOfTokens++;
            it.next();              
        }
 
System.out.println("Number of Tokens In File :" + numberOfTokens);
} 
catch(IOException e) 
{
System.out.println("Train Failed");
e.printStackTrace();
return;
}
}   
 
private void checkForCommonWord(String line, HashSet commonSet, HashSet uncommonHS)
{
StringTokenizer st = new StringTokenizer(line.toLowerCase(), 
"!-/.,@?><#~'��,'��� - �'�� �� :;][}{|�$%`'�/-^�&�*()1234567890-=_+*`�");        
 
while (st.hasMoreElements()) 
{
Object ob = st.nextElement();
 
if (!commonSet.contains(ob)) uncommonHS.add(ob);
}
}
 
public String returnString()
{
    return unCommonHS.toString();
}
 
public static void main(String[] args)     
{ 
        Tokeniser t = new Tokeniser();
}
}

Comwords.java (removes common words like 'the', 'a' etc)

import java.io.*; 
import java.util.*;
import javax.swing.JOptionPane;
import java.util.ArrayList;
import java.util.Hashtable;
 
class ComWords {    
  public ComWords() {}  
  public HashSet getCommonWordsHS () 
  {
    HashSet cwSet = new HashSet();    
    String fileName = "cwords.txt";    
    try 
    {
      BufferedReader bR = new BufferedReader(new FileReader(fileName));      
      String aLine=null;      
      while((aLine=bR.readLine())!= null) storeWord(cwSet, aLine);      
      bR.close();      
      return cwSet;      
    } 
    catch(IOException e)  
    {     
     System.out.println("CommonWords Failed");
     e.printStackTrace();     
     return null;
    }
  } 
  private void storeWord(HashSet set, String line) 
  {
    StringTokenizer st=new StringTokenizer(line," ");
    
    while (st.hasMoreElements()) 
    {
      set.add(st.nextElement());
    }
  }
}

Does anyone have any ideas how i might solve this problem?
Thanks

[Stan James]

Can you just test for a line of ten dashes immediately after the read - the first line inside your loop?

For a real change of direction, if you have JDK 5 look at Scanner. That could probably read one message at a time by using the dashes as a delimiter.

Any chance you'll be bitten by somebody including a line of dashes in their mail message?