Get source using Apache Httpclient

[gavin lee]

I tried to use apache httpclient to get source from http://marketinformation.natgrid.co.uk/gas/frmDataItemExplorer.aspx

The program listed below try to implement 2 steps:
1. go to http://marketinformation.natgrid.co.uk/gas/frmDataItemExplorer.aspx
2. expand Report node.
For the second step, I always get error 500.

I use temp data to get all post details, and I compared it with the system.out in my program, it is totally same; I also tried to add cookies got from the first step. I tried my best, but the return code is always 500. It drives me crazy.

Could any one give me some advices and help me figure it out, thanks a million.

package test;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.xpath.XPathAPI;

import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.w3c.dom.Document;
import org.w3c.tidy.Tidy;

import java.io.InputStream;
import java.io.PrintWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;
import java.util.HashMap;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.net.URLEncoder;


public class HttpRequest {


  private static final PrintWriter BEST_PRINT_WRITER_EVER = new PrintWriter(new OutputStream() {
    
    public void write(byte[] b, int off, int len) throws IOException {
    }
    
    public void write(byte[] b) throws IOException {
    }
    
    public void write(int b) throws IOException {
    }
  });


  public static NodeList getXPath(Node fromNode, String xPathQuery) throws Exception {
    if (null == fromNode) throw new IllegalArgumentException("Node parameter is null");
    if (null == xPathQuery) throw new IllegalArgumentException("XPath parameter is null");

    NodeList resultNodeList = null;

    resultNodeList = XPathAPI.selectNodeList(fromNode, xPathQuery);
    
    return resultNodeList;
  }

  private static String HOST_URL = "http://marketinformation.natgrid.co.uk/gas/frmDataItemExplorer.aspx";

  private static String INPUT_XPATH = "//input";

  private static final SimpleDateFormat REQUEST_DATE_FORMAT = new SimpleDateFormat(
      "dd/MM/yyyy");

  private static final String TYPE_ATTRIBUTE = "type";

  private static final String NAME_ATTRIBUTE = "name";

  private static final String VALUE_ATTRIBUTE = "value";

  private static final String TITLE_ATTRIBUTE = "title";

  private static final String CHECKBOX_INPUT = "checkbox";

  private static final String EVENT_VALIDATION = "__EVENTVALIDATION";

  private static final String VIEW_STATE = "__VIEWSTATE";

  public static void main(String[] args) {
    HttpRequest hq = new HttpRequest();
    hq.execute();
  }
  public void execute() {
    HttpClient httpClient = new HttpClient();
    GetMethod getMethod = new GetMethod(HOST_URL);
    InputStream is = null;
    try {
      getMethod.setFollowRedirects(true);
      httpClient.executeMethod(getMethod);
      if (getMethod.getStatusCode() != HttpStatus.SC_OK) {
        return;
      }

      Cookie[] inputcookies = httpClient.getState().getCookies();
      is = getMethod.getResponseBodyAsStream();

      Node root = null;

      // transform to xml using Tidy...
      try {
        Tidy tidy = new Tidy();
        tidy.setMakeClean(true);
        tidy.setXHTML(true);
        tidy.setXmlPi(true);
        tidy.setXmlPIs(true);
        tidy.setShowWarnings(false);
        tidy.setOnlyErrors(true);
        tidy.setQuiet(true);
        if (!false && true && true) tidy.setErrout(BEST_PRINT_WRITER_EVER);

        Document document = tidy.parseDOM(is, null);
        root = document.getDocumentElement();
      }
      catch (Exception e) {
        e.printStackTrace();
      }

      
      NodeList input = getXPath(root, INPUT_XPATH);
      Map<String, String> params = new HashMap<String, String>();
      params.put("__EVENTTARGET", "");
      params.put("__EVENTARGUMENT", "");
      params.put("tvDataItem_SelectedNode", "");
      params.put("hdnIsAddToList", "");
      params.put("tvDataItem_PopulateLog", "");
      params.put("ctrlDateTime$txtSpecifyFromDate", REQUEST_DATE_FORMAT.format(new Date()));
      params.put("ctrlDateTime$txtSpecifyToDate", REQUEST_DATE_FORMAT.format(new Date()));

      params.put("txtSearch", "");
      params.put("ctrlDateTime$rgApplicable", "rdoApplicableAt");
      params.put("ctrlDateTime$rgSpecifyDateTime", "rdoDateAndTime");


      for (int i = 0; i < input.getLength(); ++i) {
        Node n = input.item(i);
        NamedNodeMap attributes = n.getAttributes();
        String type = attributes.getNamedItem(TYPE_ATTRIBUTE).getNodeValue();
        String name = attributes.getNamedItem(NAME_ATTRIBUTE).getNodeValue();
        String value = attributes.getNamedItem(VALUE_ATTRIBUTE) == null
            ? null
            : attributes.getNamedItem(VALUE_ATTRIBUTE).getNodeValue();
                    
        if (name.equals(EVENT_VALIDATION) || name.equals(VIEW_STATE)) {
          params.put(name, value);
        }
      }

      params.put("tvDataItem_ExpandState", "cccccccccccccccc");
      params.put("__CALLBACKID", "tvDataItem");
      params.put("__CALLBACKPARAM", "11|16|fff|6|Report0|UK.Gas.Report");

      getMethod.releaseConnection();

      PostMethod postMethod = new PostMethod(HOST_URL);
      for (Map.Entry<String, String> ent : params.entrySet()) {
        String key = ent.getKey();
        String value = URLEncoder.encode(ent.getValue(), "UTF-8");
        System.out.println(key + "=" + value);
        postMethod.addParameter(key, value);
      }

      postMethod.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
      HttpState initialState = new HttpState();
      Cookie cookie = new Cookie();
      for ( Cookie c : inputcookies) {
        cookie.setDomain(c.getDomain());
        cookie.setPath(c.getPath());
        cookie.setName(c.getName());
        cookie.setValue(c.getValue());
        initialState.addCookie(cookie);
      }
      httpClient.setState(initialState);

      int returnCode = httpClient.executeMethod(postMethod);      
      if (returnCode == HttpStatus.SC_OK) {
        is = postMethod.getResponseBodyAsStream();
      }

      postMethod.releaseConnection();
    }
    catch (Exception e) {
      e.printStackTrace();
    }
    finally {
      if (is != null) {
        try {
          is.close();
        }
        catch (IOException e) {          
        }
      }
    }
  }
}

[Martijn Verburg]

Hi there and welcome to Javaranch! Hint: What is the 500 code trying to tell you?

[gavin lee]

Martijn Verburg wrote:Hi there and welcome to Javaranch! Hint: What is the 500 code trying to tell you?

Server Error in '/Gas' Application.
Runtime Error
Description: An application error occurred on the server. The current custom error settings for this application prevent the details of the application error from being viewed remotely (for security reasons). It could, however, be viewed by browsers running on the local server machine.

Details: To enable the details of this specific error message to be viewable on remote machines, please create a <customErrors> tag within a "web.config" configuration file located in the root directory of the current web application. This <customErrors> tag should then have its "mode" attribute set to "Off".

<!-- Web.Config Configuration File -->

<configuration>
<system.web>
<customErrors mode="Off"/>
</system.web>
</configuration>


Notes: The current error page you are seeing can be replaced by a custom error page by modifying the "defaultRedirect" attribute of the application's <customErrors> configuration tag to point to a custom error page URL.

<!-- Web.Config Configuration File -->

<configuration>
<system.web>
<customErrors mode="RemoteOnly" defaultRedirect="mycustompage.htm"/>
</system.web>
</configuration>

[Martijn Verburg]

You need to get hold of the error logs on the server and/or check with their developers.

[gavin lee]

Martijn Verburg wrote:You need to get hold of the error logs on the server, check with their developers.

unfortunately, I can't get the error logs on the server.

[Paul Clapham]

gavin lee wrote:

Martijn Verburg wrote:You need to get hold of the error logs on the server, check with their developers.

unfortunately, I can't get the error logs on the server.

Then do the other thing which Martijn suggested.