WikiCrawler/Backend/LightweightRobots.cs
/// <summary>
/// Return true if the current User-agent is permitted to access the given file, otherwise false.
/// throws: ArgumentException if host doesn't match host of the robot.txt file used
/// </summary>
Return true if the current User-agent is permitted to access the given file, otherwise false.
throws: ArgumentException if host doesn't match host of the robot.txt file used
Return true if the current User - agent is permitted to access the given file otherwise false . throws : ArgumentException if host doesn ' t match host of the robot . txt file used
public bool IsAllowed(Uri path)
        {
            if (!path.Host.Equals(host.Host))
            {
                throw new ArgumentException("The host of the specified URI is not the same as the robots.txt host.\n");
            }
            else
            {
                foreach (KeyValuePair<string, HashTreeNode> kv in allow)
                    if (kv.Key.Contains(UserAgent)) return kv.Value.Traverse(path.PathAndQuery);
                return allow["*"].Traverse(String.Join("", path.Segments));
            }
        }
public bool IsAllowed ( Uri path ) { if ( ! path . Host . Equals ( host . Host ) ) { throw new ArgumentException ( " \n " ) ; } else { foreach ( KeyValuePair < string , HashTreeNode > kv in allow ) if ( kv . Key . Contains ( UserAgent ) ) return kv . Value . Traverse ( path . PathAndQuery ) ; return allow [ " " ] . Traverse ( String . Join ( " " , path . Segments ) ) ; } }
Return true if the current User-agent is permitted to access the given file, otherwise false.
Return true if the current User - agent is permitted to access the given file otherwise false .
path
Uri