dolibarr/dev/tools/spider.php

#!/usr/bin/env php
<?php
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <https://www.gnu.org/licenses/>.
 */

/**
 * \file    dev/tools/spider.php
 * \brief   Script to spider Dolibarr app.
 *
 * To use it:
 * - Disable module "bookmark"
 * - Exclude param  optioncss, token, sortfield, sortorder
 */

$crawledLinks = array();
const MAX_DEPTH = 2;


/**
 * @param  string $url   URL
 * @param  string $depth Depth
 * @return string       String
 */
function followLink($url, $depth = 0)
{
	global $crawledLinks;
	$crawling = array();
	if ($depth > MAX_DEPTH) {
		echo "<div style='color:red;'>The Crawler is giving up!</div>";
		return;
	}
	$options = array(
		'http' => array(
			'method' => "GET",
			'user-agent' => "gfgBot/0.1\n"
		)
	);
	$context = stream_context_create($options);
	$doc = new DomDocument();
	@$doc->loadHTML(file_get_contents($url, false, $context));
	$links = $doc->getElementsByTagName('a');
	$pageTitle = getDocTitle($doc, $url);
	$metaData = getDocMetaData($doc);
	foreach ($links as $i) {
		$link = $i->getAttribute('href');
		if (ignoreLink($link)) {
			continue;
		}
		$link = convertLink($url, $link);
		if (!in_array($link, $crawledLinks)) {
			$crawledLinks[] = $link;
			$crawling[] = $link;
			insertIntoDatabase($link, $pageTitle, $metaData, $depth);
		}
	}
	foreach ($crawling as $crawlURL) {
		followLink($crawlURL, $depth + 1);
	}
}

/**
 * @param  string $site Site
 * @param  string $path Path
 * @return string       String
 */
function convertLink($site, $path)
{
	if (substr_compare($path, "//", 0, 2) == 0) {
		return parse_url($site)['scheme'].$path;
	} elseif (substr_compare($path, "http://", 0, 7) == 0
		or substr_compare($path, "https://", 0, 8) == 0
		or substr_compare($path, "www.", 0, 4) == 0
	) {
		return $path;
	} else {
		return $site.'/'.$path;
	}
}

/**
 * @param  string $url URL
 * @return boolean
 */
function ignoreLink($url)
{
	return $url[0] == "#" or substr($url, 0, 11) == "javascript:";
}

/**
 * @param  string $link     URL
 * @param  string $title    Title
 * @param  string $metaData Array
 * @param  int    $depth    Depth
 * @return void
 */
function insertIntoDatabase($link, $title, &$metaData, $depth)
{
	//global $crawledLinks;

	echo "Inserting new record {URL= ".$link.", Title = '$title', Description = '".$metaData['description']."', Keywords = ' ".$metaData['keywords']."'}<br/><br/><br/>";

	//²$crawledLinks[]=$link;
}

/**
 * @param  string $doc Doc
 * @param  string $url URL
 * @return string               URL/Title
 */
function getDocTitle(&$doc, $url)
{
	$titleNodes = $doc->getElementsByTagName('title');
	if (count($titleNodes) == 0 or !isset($titleNodes[0]->nodeValue)) {
		return $url;
	}
	$title = str_replace('', '\n', $titleNodes[0]->nodeValue);
	return (strlen($title) < 1) ? $url : $title;
}

/**
 * @param  string $doc Doc
 * @return array                Array
 */
function getDocMetaData(&$doc)
{
	$metaData = array();
	$metaNodes = $doc->getElementsByTagName('meta');
	foreach ($metaNodes as $node) {
		$metaData[$node->getAttribute("name")] = $node->getAttribute("content");
	}
	if (!isset($metaData['description'])) {
		$metaData['description'] = 'No Description Available';
	}
	if (!isset($metaData['keywords'])) {
		$metaData['keywords'] = '';
	}
	return array(
		'keywords' => str_replace('', '\n', $metaData['keywords']),
		'description' => str_replace('', '\n', $metaData['description'])
	);
}


followLink("http://localhost/dolibarr_dev/htdocs");