11/25/2013 5:59:36 PM

Using the .NET Framework and ASP.NET, the best way to determine if an incoming web request is a search bot, we must look at the UserAgent string in the request header. The code below is a simple way to determine most bot requests. The user agent list can be updated to track more requests (http://www.useragentstring.com/pages/Crawlerlist/).

public static bool IsRequestSearchBot() { if (System.Web.HttpContext.Current != null && System.Web.HttpContext.Current.Request != null) { string userAgent = System.Web.HttpContext.Current.Request.UserAgent; if (string.IsNullOrEmpty(userAgent)) { return false; } userAgent = userAgent.ToLower(); string[] userAgents = new string[] { "googlebot", "bingbot", "msnbot", "yahoo! slurp", "baiduspider", "iaskspider", "ask jeeves" }; foreach (string agent in userAgents) { if (userAgent.Contains(agent)) { return true; } } } return false; }