I need to write an app that monitors 1000+ sites every 5
minutes to see if they're still up. I'm using the code
below (extracted from a larger generic http class we use)
to do this. I launch each process as a new thread
because, if you give a site 5-10 seconds to time out,
doing 1000+ sites would take around an hour to process, so
it needs to be asynch. This craps out on me after about
80 threads giving me stream errors. Anyone have a better
suggestion of how to approach something like this in .NET?
public string GetHttpString(string sURL)
{
debugPrint("GetHttpsString: sURL=" + sURL);
this.url=sURL;
this.httpMethod =
utils.clsHTTPS.HttpMethodValues.GET;
this.contentType = "text/html";
System.IO.MemoryStream stream = GetHttpStream();
string sRes = ConvertMemoryStreamToString(stream);
if (this.outputFolder != "")
{
utils.clsFile f = new clsFile();
f.stringToFile(this.outputFolder + "res" +
this.iOutputCount.ToString() + ".htm", sRes);
this.iOutputCount++;
}
return sRes;
}
public System.IO.MemoryStream GetHttpStream()
{
int iRetryCount = 0;
while (iRetryCount < this.retryCount | iRetryCount
== 0)
{
try
{
System.Net.HttpWebRequest request
= (System.Net.HttpWebRequest) System.Net.WebRequest.Create
(this.url);
request.CookieContainer =
this.CookieContainer;
System.Type methType =
this.HttpMethod.GetType();
request.Method =
utils.clsHTTPS.HttpMethodValues.GetName(methType,
this.HttpMethod);
if(this.ProxyAddress!="" &&
this.ProxyAddress!=null)
{
System.Net.WebProxy
myProxy = new System.Net.WebProxy(this.ProxyAddress,
this.ProxyPort);
myProxy.BypassProxyOnLocal
= true;
request.Proxy = myProxy;
}
else
{
request.Proxy =
WebProxy.GetDefaultProxy();
}
if(this.Referer!="" &&
this.Referer!=null) request.Referer = this.Referer;
certColl=request.ClientCertificates;
request.AllowAutoRedirect =
this.allowAutoRedirect;
request.ContentType=this.contentType;
request.KeepAlive=this.keepAlive;
request.Timeout = this.timeout;
request.UserAgent = this.userAgent;
if
(this.HttpMethod==utils.clsHTTPS.HttpMethodValues.POST)
{
if(this.Headers.Count>0)
request.Headers = this.Headers;;
System.IO.Stream
sendStream;
byte [] bytes = null;
// Get the data that is
being posted (or sent) to the server
bytes =
System.Text.Encoding.ASCII.GetBytes(this.PostParameters);
request.ContentLength =
bytes.Length;
sendStream =
request.GetRequestStream();
sendStream.Write
(bytes,0,bytes.Length);
sendStream.Close();
}
if(this.Username!="" &&
this.Username!=null)
{
request.Credentials = new
System.Net.NetworkCredential(this.Username,this.Password);
}
System.Net.WebResponse response =
request.GetResponse();
System.IO.Stream stream =
(System.IO.Stream)response.GetResponseStream();
System.IO.MemoryStream ms = new
System.IO.MemoryStream(ConvertStreamToByteBuffer(stream));
//iRetryCount = 3;
return ms;
}
catch (System.Net.WebException webE)
{
debugPrint(webE.Message);
debugPrint("Retry " +
iRetryCount.ToString());
iRetryCount++;
for (int i = 0; i <
this.retrySeconds; i++)
System.Threading.Thread.Sleep(1000);
}
finally
{
}
}
debugPrint("Retried " + iRetryCount.ToString() + "
times - bailing");
return null;
}
public static string ConvertMemoryStreamToString
(System.IO.MemoryStream theStream)
{
try
{
byte[] data = new byte[theStream.Length];
theStream.Read(data, 0, data.Length);
return
System.Text.ASCIIEncoding.ASCII.GetString(data);
}
catch
{
return "";
}
}
