HttpUrlConnection: Login works only for first page

[Daniela Berger]

Hi, everyone

I have a rather vexing problem:
My current project is a web agent that is supposed to log into a website, do stuff, and log off again.
Now, I managed to download the login page and I know I managed to login, because I can download the next page.
But then, if I try to "do stuff", that is, if I try to download the next page, I get a page that basically says "You're already logged in".
I tried this with the page that the web agent will work on, and since it didn't work, I tried it on our web site. But still the same problem:

1. I download the login page (which is not strictly necessary)
2. I send the login data and receive the first "logged-in" page
3. I try to download the next page and get our page that tells me that I'm already logged in

That is a major problem, because that keeps me from logging off properly.

I include both the Java code and a log file from Firefox's LiveHTTP module.
And now for some code:

public class WebAgent
{
static Hashtable theCookies = new Hashtable();
public String PHPSESSID = "";

  public WebAgent(String input) 
  {
  this.PHPSESSID = input;
  try
  {
    go_fiffy_go();
  }
catch (MalformedURLException e1) 
{
System.out.print(e1 + this.cr);
}
catch (IOException e2) 
{
System.out.print(e2 + this.cr);
}
catch (XPathException e3) 
{
System.out.print(e3 + this.cr);
}
  }
  
  public void go_fiffy_go() throws IOException, MalformedURLException, XPathException 
  {
  
/* v v v 1st Connection: demand login page v v v */
 
    URL targetURL = new URL("http://www.cultuzz.com");
    HttpURLConnection targetConnection = (HttpURLConnection) targetURL.openConnection();
 
    targetConnection.setDoOutput(true);
    targetConnection.setDoInput(true);
    targetConnection.setRequestMethod("GET");
    targetConnection.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0");
    targetConnection.setRequestProperty("Accept", "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
    targetConnection.setRequestProperty("Accept-Language", "de-de,de;q=0.8,en-us;q=0.5,en;q=0.3");
    targetConnection.setRequestProperty("Accept-Encoding", "gzip,deflate");
    targetConnection.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
    targetConnection.setRequestProperty("Keep-Alive", "300");
    targetConnection.setRequestProperty("Connection", "keep-alive");
    targetConnection.setRequestProperty("Referer", "Referer: [url=http://www.cultuzz.com/index.php");]http://www.cultuzz.com/index.php");[/url]
    
    // read the file from the connection and save to a file
    readFromURL(targetConnection, "/www/htdocs/cultagent/log/" + getFileTime() + "-Login01.txt");
    targetConnection.disconnect();
/* ^ ^ ^ 1st Connection: demand login page ^ ^ ^ */
 
/* v v v 2nd Connection: call login page, send cookie, send POST vars  v v v */
/* v v v                 save output                                   v v v */
    URL targetURL2 = new URL("https://albatros.cultuzz.de/service/index.php");
    HttpURLConnection targetConnection2 = (HttpURLConnection) targetURL2.openConnection();
    if (targetConnection2 instanceof HttpsURLConnection) {
      try {
      KeyManager[] km = null;
      TrustManager[] tm = {new RelaxedX509TrustManager()};
      SSLContext sslContext = SSLContext.getInstance("SSL");
      sslContext.init(null, tm, new java.security.SecureRandom());
      SSLSocketFactory sf = sslContext.getSocketFactory();
      ((HttpsURLConnection)targetConnection2).setSSLSocketFactory(sf);
      }
      catch (java.security.GeneralSecurityException e) { throw new IOException(e.getMessage()); }
      }    
    targetConnection2.setDoOutput(true);
    targetConnection2.setDoInput(true);
    targetConnection2.setRequestMethod("POST");
    targetConnection2.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0");
    targetConnection2.setRequestProperty("Accept", "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
    targetConnection2.setRequestProperty("Accept-Language", "de-de,de;q=0.8,en-us;q=0.5,en;q=0.3");
    targetConnection2.setRequestProperty("Accept-Encoding", "gzip,deflate");
    targetConnection2.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
    targetConnection2.setRequestProperty("Keep-Alive", "300");
    targetConnection2.setRequestProperty("Connection", "keep-alive");
    targetConnection2.setRequestProperty("Referer", "http://www.cultuzz.com/index.php");
    targetConnection2.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
    // apparently the login page sends the PHP session id as a cookie
    targetConnection2.setRequestProperty("Set-Cookie", "PHPSESSID=" + this.PHPSESSID);
    
    OutputStreamWriter wr = new OutputStreamWriter(targetConnection2.getOutputStream());
    wr.write("benutzer_kennung=XXXXX&benutzer_passwort=YYYYY&aktion=login&aktion=login&sprach_id_webseite=1");
    wr.flush();
     wr.close();    
    
    // process headers and extract cookies
    saveCookies(targetConnection2);
    
    // save the output to a file
    readFromURL(targetConnection2, "/www/htdocs/cultagent/log/" + getFileTime() + "-Login02.txt");
    targetConnection2.disconnect();    
/* ^ ^ ^ 2nd Connection: call login page, send cookie, send POST vars  ^ ^ ^ */
/* ^ ^ ^                 save output                                   ^ ^ ^ */
 
/* v v v 3rd Connection: access any page while logged on  v v v */
    URL targetURL3 = new URL("https://albatros.cultuzz.de/service/");
    HttpsURLConnection targetConnection3 = (HttpsURLConnection) targetURL3.openConnection();    
    if (targetConnection3 instanceof HttpsURLConnection) {
      try {
      KeyManager[] km = null;
      TrustManager[] tm = {new RelaxedX509TrustManager()};
      SSLContext sslContext = SSLContext.getInstance("SSL");
      sslContext.init(null, tm, new java.security.SecureRandom());
      SSLSocketFactory sf = sslContext.getSocketFactory();
      ((HttpsURLConnection)targetConnection3).setSSLSocketFactory(sf);
      }
      catch (java.security.GeneralSecurityException e) { throw new IOException(e.getMessage()); }
      }    
    targetConnection3.setDoOutput(true);
    targetConnection3.setDoInput(true);
    targetConnection3.setRequestMethod("GET");
    targetConnection3.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0");
    targetConnection3.setRequestProperty("Accept", "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
    targetConnection3.setRequestProperty("Accept-Language", "de-de,de;q=0.8,en-us;q=0.5,en;q=0.3");
    targetConnection3.setRequestProperty("Accept-Encoding", "gzip,deflate");
    targetConnection3.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
    targetConnection3.setRequestProperty("Keep-Alive", "300");
    targetConnection3.setRequestProperty("Connection", "keep-alive");
    targetConnection3.setRequestProperty("Referer", "https://albatros.cultuzz.de/service/index.php");
    targetConnection3.setRequestProperty("Set-Cookie", returnCookies());
    
    readFromURL(targetConnection3, "/www/htdocs/cultagent/log/" + getFileTime() + "-Login03.txt");
    targetConnection3.disconnect();  
/* ^ ^ ^ 3rd Connection: access any page while logged on  ^ ^ ^ */    
 
/* v v v 4th Connection: log off  v v v */    
    URL targetURL4 = new URL("https://albatros.cultuzz.de/service/index.php?aktion=logout&username=");
    HttpsURLConnection targetConnection4 = (HttpsURLConnection) targetURL4.openConnection();
    if (targetConnection4 instanceof HttpsURLConnection) {
      try {
      KeyManager[] km = null;
      TrustManager[] tm = {new RelaxedX509TrustManager()};
      SSLContext sslContext = SSLContext.getInstance("SSL");
      sslContext.init(null, tm, new java.security.SecureRandom());
      SSLSocketFactory sf = sslContext.getSocketFactory();
      ((HttpsURLConnection)targetConnection4).setSSLSocketFactory(sf);
      }
      catch (java.security.GeneralSecurityException e) { throw new IOException(e.getMessage()); }
      }    
    targetConnection4.setDoOutput(true);
    targetConnection4.setDoInput(true);
    targetConnection4.setRequestMethod("GET");
    targetConnection4.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0");
    targetConnection4.setRequestProperty("Accept", "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5");
    targetConnection4.setRequestProperty("Accept-Language", "de-de,de;q=0.8,en-us;q=0.5,en;q=0.3");
    targetConnection4.setRequestProperty("Accept-Encoding", "gzip,deflate");
    targetConnection4.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
    targetConnection4.setRequestProperty("Keep-Alive", "300");
    targetConnection4.setRequestProperty("Connection", "keep-alive");
    targetConnection4.setRequestProperty("Referer", "https://albatros.cultuzz.de/service/index.php");
    targetConnection4.setRequestProperty("Set-Cookie", returnCookies());
    
    readFromURL(targetConnection4, "/www/htdocs/cultagent/log/" + getFileTime() + "-Login04.txt");
    targetConnection4.disconnect();  
/* ^ ^ ^ 4th Connection: log off  ^ ^ ^ */    
  }
 
/* v v v all kinds of methods, probably not all that interesting...  v v v */    
  
  class RelaxedX509TrustManager implements X509TrustManager 
  {
    public boolean isClientTrusted(java.security.cert.X509Certificate[] chain){ return true; }
    public boolean isServerTrusted(java.security.cert.X509Certificate[] chain){ return true; }
    public java.security.cert.X509Certificate[] getAcceptedIssuers() { return null; }
    public void checkClientTrusted(java.security.cert.X509Certificate[] chain, String input) {}
    public void checkServerTrusted(java.security.cert.X509Certificate[] chain, String input) {}
  }
  
  public String getFileTime()
  {
    String returnString = "";
    SimpleDateFormat formatter = new SimpleDateFormat("H_mm_ss", Locale.GERMAN);
    returnString = formatter.format(new Date());
    return returnString;
  }
  
  public void writeToFile(String inputString, String fileName)
  {
    try 
    {
//      File outputFile = new File(fileName);
      BufferedWriter out = new BufferedWriter(new FileWriter(fileName));
      out.write(inputString);
      out.close();
    } 
    catch (IOException e) 
    {
      System.out.print(e + this.cr);
    }
  }
  
  public String readFromURL(URLConnection connection, String logFile)
  {
    String outputString = ""; 
    outputString = getHeaderFields(connection);
    try
    {
      BufferedReader rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
      String inputLine = "";
     
      while ((inputLine = rd.readLine()) != null) 
      {
        outputString = outputString + inputLine + "\n";
      }
      rd.close();
    }
    catch(IOException e)
    {
      System.out.print(e + this.cr);
    }
    if(!logFile.equals(""))
    {
      writeToFile(outputString, logFile);
    }
    return outputString;
  }
  
  public String getHeaderFields(URLConnection connection)
  {
    String outputString = "";
    String headerFields = connection.getHeaderFields().toString();
    
    outputString = headerFields.replaceAll("], ", "], \n");
    return outputString;
  }
  
  public void saveCookies(URLConnection connection)
  {
    for (int i=0; ; i++) 
    {
      String headerName = connection.getHeaderFieldKey(i);
      String headerValue = connection.getHeaderField(i);
 
      if (headerName == null && headerValue == null) 
      {
        // No more headers
        break;
      }
      if ("Set-Cookie".equalsIgnoreCase(headerName)) 
      {
        // Parse cookie
        String[] fields = headerValue.split(";\\s*");
        String cookieValue = fields[0];
        String cookieValueName = "";
        String cookieValueValue = "";
        if(cookieValue.indexOf("=") != -1)
        {
          String[] sessionId = cookieValue.split("=");
          cookieValueName = sessionId[0];
          cookieValueValue = sessionId[1];
        }
        else
        {
          cookieValueName = "cookieValue";
          cookieValueValue = cookieValue;
        }
        theCookies.put(cookieValueName, cookieValueValue);
        String expires = null;
        String path = null;
        String domain = null;
        boolean secure = false;
 
        // Parse each field
        for (int j=1; j<fields.length; j++) 
        {
          if ("secure".equalsIgnoreCase(fields[j])) 
          {
            secure = true;
          } 
          else if (fields[j].indexOf('=') > 0) 
          {
            String[] f = fields[j].split("=");
            if ("expires".equalsIgnoreCase(f[0])) 
            {
              expires = f[1];
              theCookies.put("expires", expires);
            } 
            else if ("domain".equalsIgnoreCase(f[0])) 
            {
              domain = f[1];
              theCookies.put("domain", domain);
            } 
            else if ("path".equalsIgnoreCase(f[0])) 
            {
              path = f[1];
//              theCookies.put("path", path);
            }
          }
        }
        System.out.print("Save Cookie: " + cookieValue 
            + "; expires=" + expires 
             + "; path=" + path
             + "; domain=" + domain + "; "
             + secure + this.cr);
      }
    }
  }
  
  public String returnCookies()
  {
    String returnString = "";
    Iterator it = theCookies.keySet().iterator();
    while(it.hasNext())
    {
      Object key = (String) it.next();
      returnString = returnString + "�" + key + "=" + theCookies.get(key) + "�"; 
    }
    
    returnString = returnString.replaceAll("��", ";");
    returnString = returnString.replaceAll("�", "");
    
    System.out.print("Return Cookie: " + returnString + this.cr);
    return returnString;
  }
  
  public static void main(String[] args) 
  {
    System.out.println("start");
    WebAgent wa = new WebAgent(args[0]);
    System.out.println("end" + wa.cr);
  }
}

Just in case anybody is wondering, this is what the LiveHTTP module for the FireFox logs when I log in "manually"

----------------------------------------------------------
http://www.cultuzz.com/index.php

GET /index.php HTTP/1.1
Host: www.cultuzz.com
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip,deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 300
Connection: keep-alive

HTTP/1.x 200 OK
Date: Thu, 01 Dec 2005 09:46:40 GMT
Server: Apache/1.3.26 (Linux/SuSE) mod_ssl/2.8.10 OpenSSL/0.9.6g PHP/4.3.3
X-Powered-By: PHP/4.3.3
Keep-Alive: timeout=15, max=98
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html
----------------------------------------------------------
https://albatros.cultuzz.de/service/index.php

POST /service/index.php HTTP/1.1
Host: albatros.cultuzz.de
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip,deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 300
Connection: keep-alive
Referer: http://www.cultuzz.com/index.php
Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd
Content-Type: application/x-www-form-urlencoded
Content-Length: 97
benutzer_kennung=XXXXX&benutzer_passwort=YYYYY&aktion=login&aktion=login&sprach_id_webseite=1
HTTP/1.x 200 OK
Date: Thu, 01 Dec 2005 09:46:43 GMT
Server: Apache/1.3.26 (Linux/SuSE) mod_ssl/2.8.10 OpenSSL/0.9.6g PHP/4.3.3
X-Powered-By: PHP/4.3.3
Set-Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd; path=/
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Keep-Alive: timeout=15, max=100
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html
----------------------------------------------------------
https://albatros.cultuzz.de/service/_template.php //... snip

GET /service/_template.php //... snip
Host: albatros.cultuzz.de
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip,deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 300
Connection: keep-alive
Referer: https://albatros.cultuzz.de/service/index.php
Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd

HTTP/1.x 200 OK
Date: Thu, 01 Dec 2005 09:46:47 GMT
Server: Apache/1.3.26 (Linux/SuSE) mod_ssl/2.8.10 OpenSSL/0.9.6g PHP/4.3.3
X-Powered-By: PHP/4.3.3
Set-Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd; path=/
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Keep-Alive: timeout=15, max=99
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html
----------------------------------------------------------
https://albatros.cultuzz.de/service/index.php?aktion=logout&username=

GET /service/index.php?aktion=logout&username= HTTP/1.1
Host: albatros.cultuzz.de
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip,deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 300
Connection: keep-alive
Referer: https://albatros.cultuzz.de/service/_template.php //... snip
Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd

HTTP/1.x 302 Found
Date: Thu, 01 Dec 2005 09:46:51 GMT
Server: Apache/1.3.26 (Linux/SuSE) mod_ssl/2.8.10 OpenSSL/0.9.6g PHP/4.3.3
X-Powered-By: PHP/4.3.3
Set-Cookie: PHPSESSID=ef95c17d3d442f70769e7be9bdbd40dd; path=/
Expires: Thu, 19 Nov 1981 08:52:00 GMT
Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0
Pragma: no-cache
Location: http://www.cultuzz.com/index.php?meldung=
Keep-Alive: timeout=15, max=98
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html
----------------------------------------------------------
http://www.cultuzz.com/index.php?meldung=

GET /index.php?meldung= HTTP/1.1
Host: www.cultuzz.com
User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0
Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
Accept-Language: de-de,de;q=0.8,en-us;q=0.5,en;q=0.3
Accept-Encoding: gzip,deflate
Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
Keep-Alive: 300
Connection: keep-alive

HTTP/1.x 200 OK
Date: Thu, 01 Dec 2005 09:46:51 GMT
Server: Apache/1.3.26 (Linux/SuSE) mod_ssl/2.8.10 OpenSSL/0.9.6g PHP/4.3.3
X-Powered-By: PHP/4.3.3
Keep-Alive: timeout=15, max=97
Connection: Keep-Alive
Transfer-Encoding: chunked
Content-Type: text/html
----------------------------------------------------------