shop.balmet.com

robotstxtparser.php (9340B)

---

 <?php
 /**
  * Class for parsing robots.txt files
  *
  * @author Eugene Yurkevich (bopodaa@gmail.com)
  *
  *
  * Some useful links and materials:
  * @link https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt
  * @link https://help.yandex.com/webmaster/controlling-robot/robots-txt.xml
  */
  
 namespace d_robots_txt_parser;
 
 class RobotsTxtParser
 {
 	// default encoding
 	const DEFAULT_ENCODING = 'UTF-8';
 
 	// states
 	const STATE_ZERO_POINT = 'zero-point';
 	const STATE_READ_DIRECTIVE = 'read-directive';
 	const STATE_SKIP_SPACE = 'skip-space';
 	const STATE_SKIP_LINE = 'skip-line';
 	const STATE_READ_VALUE = 'read-value';
 
 	// directives
 	const DIRECTIVE_ALLOW = 'allow';
 	const DIRECTIVE_DISALLOW = 'disallow';
 	const DIRECTIVE_HOST = 'host';
 	const DIRECTIVE_SITEMAP = 'sitemap';
 	const DIRECTIVE_USERAGENT = 'user-agent';
 	const DIRECTIVE_CRAWL_DELAY = 'crawl-delay';
 	const DIRECTIVE_CLEAN_PARAM = 'clean-param';
 
 	/**
 	 * Default user-agent
 	 * First off, links should be checked by specific user-agent rules. If specific user-agent isn't specified than default user-agent used.
 	 */
 	const USER_AGENT_ALL = '*';
 
 	// current state
 	private $state = '';
 
 	// robots.txt file content
 	private $content = '';
 
 	// rules set
 	private $rules = array();
 
 	// internally used variables
 	private $current_word = '';
 	private $current_char = '';
 	private $char_index = 0;
 	private $current_directive = '';
 	private $userAgent = self::USER_AGENT_ALL;
 
 	/**
 	 * @param  string $content - file content
 	 * @param  string $encoding - encoding
 	 * @return RobotsTxtParser
 	 */
 	public function __construct($content, $encoding = self::DEFAULT_ENCODING)
 	{
 		// convert encoding
 		$encoding = !empty($encoding) ? $encoding : mb_detect_encoding($content);
 		mb_internal_encoding($encoding);
 
 		// set content
 		$this->content = iconv($encoding, 'UTF-8//IGNORE', $content);
 		$this->content .= "\n";
 
 		// set default state
 		$this->state = self::STATE_ZERO_POINT;
 
 		// parse rules - default state
 		$this->prepareRules();
 	}
 
 	/**
 	 * Get rules by specific bot (user-agent)
 	 * Use $userAgent = NULL to get all rules for all user-agents grouped by user-agent. User-agents will return in lower case.
 	 * Use $userAgent = '*' to get common rules.
 	 * Use $userAgent = 'YandexBot' to get rules for user-agent 'YandexBot'.
 	 *
 	 * @param string $userAgent
 	 * @return array
 	 */
 	public function getRules($userAgent = NULL)
 	{
 		if (is_null($userAgent)) {
 			//return all rules
 			return $this->rules;
 		}
 		else {
 			$userAgent = mb_strtolower($userAgent);
 			if (isset($this->rules[$userAgent])) {
 				return $this->rules[$userAgent];
 			}
 			else {
 				return array();
 			}
 		}
 	}
 
 	/**
 	 * Get sitemaps links.
 	 * Sitemap always relates to all user-agents and return in rules with user-agent "*"
 	 *
 	 * @return array  all sitemap urls
 	 */
 	public function getSitemaps()
 	{
 		$rules = $this->getRules(self::USER_AGENT_ALL);
 		if (!empty($rules[self::DIRECTIVE_SITEMAP])) {
 			return $rules[self::DIRECTIVE_SITEMAP];
 		}
 
 		return array();
 	}
 
 	public function getContent()
 	{
 		return $this->content;
 	}
 
 	/**
 	 * Comment signal (#)
 	 */
 	private function sharp()
 	{
 		return ($this->current_char == '#');
 	}
 
 	/**
 	 * Allow directive signal
 	 */
 	private function directiveAllow()
 	{
 		return ($this->current_directive == self::DIRECTIVE_ALLOW);
 	}
 
 	/**
 	 * Disallow directive signal
 	 */
 	private function directiveDisallow()
 	{
 		return ($this->current_directive == self::DIRECTIVE_DISALLOW);
 	}
 
 	/**
 	 * Host directive signal
 	 */
 	private function directiveHost()
 	{
 		return ($this->current_directive == self::DIRECTIVE_HOST);
 	}
 
 	/**
 	 * Sitemap directive signal
 	 */
 	private function directiveSitemap()
 	{
 		return ($this->current_directive == self::DIRECTIVE_SITEMAP);
 	}
 
 	/**
 	 * Clean-param directive signal
 	 */
 	private function directiveCleanParam()
 	{
 		return ($this->current_directive == self::DIRECTIVE_CLEAN_PARAM);
 	}
 
 	/**
 	 * User-agent directive signal
 	 */
 	private function directiveUserAgent()
 	{
 		return ($this->current_directive == self::DIRECTIVE_USERAGENT);
 	}
 
 	/**
 	 * Crawl-Delay directive signal
 	 */
 	private function directiveCrawlDelay()
 	{
 		return ($this->current_directive == self::DIRECTIVE_CRAWL_DELAY);
 	}
 
 	/**
 	 * Key : value pair separator signal
 	 */
 	private function lineSeparator()
 	{
 		return ($this->current_char == ':');
 	}
 
 	/**
 	 * Move to new line signal
 	 */
 	private function newLine()
 	{
 		$asciiCode = ord($this->current_char);
 
 		return ($this->current_char == "\n"
 			|| $asciiCode == 13
 			|| $asciiCode == 10
 			|| $this->current_word == "\r\n"
 			|| $this->current_word == "\n\r"
 		);
 	}
 
 	/**
 	 * "Space" signal
 	 */
 	private function space()
 	{
 		return ($this->current_char == "\s");
 	}
 
 	/**
 	 * Change state
 	 *
 	 * @param string $stateTo - state that should be set
 	 * @return void
 	 */
 	private function switchState($stateTo = self::STATE_SKIP_LINE)
 	{
 		$this->state = $stateTo;
 	}
 
 	/**
 	 * Parse rules
 	 *
 	 * @return void
 	 */
 	public function prepareRules()
 	{
 		$contentLength = mb_strlen($this->content);
 		while ($this->char_index <= $contentLength) {
 			$this->step();
 		}
 
 		foreach ($this->rules as $userAgent => $directive) {
 			foreach ($directive as $directiveName => $directiveValue) {
 				if (is_array($directiveValue)) {
 					$this->rules[$userAgent][$directiveName] = array_values(array_unique($directiveValue));
 				}
 			}
 		}
 	}
 
 	/**
 	 * Check if we should switch
 	 * @return bool
 	 */
 	private function shouldSwitchToZeroPoint()
 	{
 		return in_array(strtolower($this->current_word), array(
 			self::DIRECTIVE_ALLOW,
 			self::DIRECTIVE_DISALLOW,
 			self::DIRECTIVE_HOST,
 			self::DIRECTIVE_USERAGENT,
 			self::DIRECTIVE_SITEMAP,
 			self::DIRECTIVE_CRAWL_DELAY,
 			self::DIRECTIVE_CLEAN_PARAM,
 		), true);
 	}
 
 	/**
 	 * Process state ZERO_POINT
 	 * @return RobotsTxtParser
 	 */
 	private function zeroPoint()
 	{
 		if ($this->shouldSwitchToZeroPoint()) {
 			$this->switchState(self::STATE_READ_DIRECTIVE);
 		} // unknown directive - skip it
 		elseif ($this->newLine()) {
 			$this->current_word = "";
 			$this->increment();
 		}
 		else {
 			$this->increment();
 		}
 		return $this;
 	}
 
 	/**
 	 * Read directive
 	 * @return RobotsTxtParser
 	 */
 	private function readDirective()
 	{
 		$this->current_directive = strtolower(trim($this->current_word));
 
 		$this->increment();
 
 		if ($this->lineSeparator()) {
 			$this->current_word = "";
 			$this->switchState(self::STATE_READ_VALUE);
 		}
 		else {
 			if ($this->space()) {
 				$this->switchState(self::STATE_SKIP_SPACE);
 			}
 			if ($this->sharp()) {
 				$this->switchState(self::STATE_SKIP_LINE);
 			}
 		}
 		return $this;
 	}
 
 	/**
 	 * Skip space
 	 * @return RobotsTxtParser
 	 */
 	private function skipSpace()
 	{
 		$this->char_index++;
 		$this->current_word = mb_substr($this->current_word, -1);
 		return $this;
 	}
 
 	/**
 	 * Skip line
 	 * @return RobotsTxtParser
 	 */
 	private function skipLine()
 	{
 		$this->char_index++;
 		$this->switchState(self::STATE_ZERO_POINT);
 		return $this;
 	}
 
 	/**
 	 * Read value
 	 * @return RobotsTxtParser
 	 */
 	private function readValue()
 	{
 		if ($this->newLine()) {
 			$this->assignValueToDirective();
 		}
 		elseif ($this->sharp()) {
 			$this->current_word = mb_substr($this->current_word, 0, -1);
 			$this->assignValueToDirective();
 		}
 		else {
 			$this->increment();
 		}
 		return $this;
 	}
 
 	private function assignValueToDirective()
 	{
 		if ($this->directiveUserAgent()) {
 			$this->userAgent = mb_strtolower(trim($this->current_word));
 			if (!isset($this->rules[$this->userAgent])) {
 				$this->rules[$this->userAgent] = array();
 			}
 		}
 		elseif ($this->directiveCrawlDelay()) {
 			$this->rules[$this->userAgent][$this->current_directive] = (double)$this->current_word;
 		}
 		elseif ($this->directiveSitemap()) {
 			$this->rules[self::USER_AGENT_ALL][$this->current_directive][] = $this->current_word;
 		}
 		elseif ($this->directiveCleanParam()) {
 			$this->rules[$this->userAgent][$this->current_directive][] = trim($this->current_word);
 		}
 		elseif ($this->directiveHost()) {
 			if (empty($this->rules['*'][$this->current_directive])) { // save only first host directive value, assign to '*'
 				$this->rules['*'][$this->current_directive] = $this->current_word;
 			}
 		}
 		else {
 			if (!empty($this->current_word)) {
 				$this->rules[$this->userAgent][$this->current_directive][] = $this->current_word;
 			}
 		}
 		$this->current_word = '';
 		$this->current_directive = '';
 		$this->switchState(self::STATE_ZERO_POINT);
 	}
 
 	/**
 	 * Machine step
 	 *
 	 * @return void
 	 */
 	private function step()
 	{
 		switch ($this->state) {
 			case self::STATE_ZERO_POINT:
 				$this->zeroPoint();
 				break;
 
 			case self::STATE_READ_DIRECTIVE:
 				$this->readDirective();
 				break;
 
 			case self::STATE_SKIP_SPACE:
 				$this->skipSpace();
 				break;
 
 			case self::STATE_SKIP_LINE:
 				$this->skipLine();
 				break;
 
 			case self::STATE_READ_VALUE:
 				$this->readValue();
 				break;
 		}
 	}
 
 	/**
 	 * Move to the following step
 	 *
 	 * @return void
 	 */
 	private function increment()
 	{
 		$this->current_char = mb_substr($this->content, $this->char_index, 1);
 		$this->current_word .= $this->current_char;
 		if (!$this->directiveCleanParam() && !$this->directiveUserAgent()) {
 			$this->current_word = trim($this->current_word);
 		}
 		$this->char_index++;
 	}
 }