/*
 * OutWit Hub
 *
 * Copyright (c) OutWit Technologies 2007-2020
 * All rights reserved
 */
/* jshint esversion: 6 */
/* jshint -W014 */
/* jshint -W084 */
/* jshint -W103 */
/* jshint -W107 */
/* jshint -W008 */
/* jshint -W004 */
/* jshint -W069*/
/* jshint -W083*/
/* jshint -W086*/
/* use moz */

/**
 * @fileOverview Extraction-related hub operations
 * @author pr
 */


/**
 * The extractor class is responsible for controlling which extractor have already been runned for
 * the current page and also control which extractors are in the autoRun mode.
 *
 * AutoRun is NOT an attribute, it is an STATE. So, you may test if the extractor is in autoRun mode
 * with isAutoRun method. The autorun mode may be affected ONLY by the properties of the extractor.
 *
 * It is important to understand the diffrence between this criteria and the bottonPanel.isDefault function.
 * Even if we could use the bottonPanel for the same controlling purposes, it is better to keep this object
 * so we will be able to add new properties without necessary adding new interface elements and try to keep
 * the logic out of the interface.
 *
 * @todo make the code more readable and take a better look in the start/stop throbber
 * @class
 * @static
 * @name outwitHub.extractor
 * @memberOf outwitHub
 */
function extractor(preDefAutoRunIn, appendOnStopIn, nameIn, viewNameIn, messageVerbosityIn, maxCountLightVersion) {

	this.lastCount = 0;

	this._autoRunProperties = {};


	this.addAutoRunProperty = function(aPropertyName, aDefaultValue) {
		this._autoRunProperties[aPropertyName] = aDefaultValue;
		this.__defineSetter__(aPropertyName, function(val) {
			this._setAndExtractorAutoRunTest(aPropertyName, val);
		});
		this.__defineGetter__(aPropertyName, function() {
			return this._autoRunProperties[aPropertyName];
		});
	};

	this.addAutoRunProperty("preDefAutoRun", preDefAutoRunIn);
	this.addAutoRunProperty("autoGrab", false);
	this.addAutoRunProperty("smartGrab", true);
	this.addAutoRunProperty("autoCatch", false);
	this.addAutoRunProperty("current", false);
	this.addAutoRunProperty("notEmpty", false);
	this.addAutoRunProperty("download", false);
	this.addAutoRunProperty("export", false);
	this.addAutoRunProperty("oneShot", false);

	/**
	 * The extractor may be set to autoRun as default. This means that it will always run when a new uri is loaded
	 * @name outwitHub.extractor#preDefAutoRun
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["preDefAutoRun"] = preDefAutoRunIn;
	   this.__defineSetter__("preDefAutoRun", function(val) {
	   this._setAndExtractorAutoRunTest("preDefAutoRun", val);
	   });
	   this.__defineGetter__("preDefAutoRun", function(){
	   return this._autoRunProperties["preDefAutoRun"];
	   });
	   */



	/**
	 * The autoCatch value should be set to true or false depending on the checkbox "autoCatch" of the views of each extractor.
	 * If set to "true", the extractor will be in autoRun mode. The test to catch or not will be done in the bottonPanel, not here, this
	 * is used only to control the autoRun.
	 * @name outwitHub.extractor#autoCatch
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["autoCatch"] = false;
	   this.__defineSetter__("autoCatch", function(val) {
	   this._setAndExtractorAutoRunTest("autoCatch", val);
	   });

	   this.__defineGetter__("autoCatch", function(){
	   return this._autoRunProperties["autoCatch"];
	   });
	   */

	/**
	 * The current value should be set to true or false depending on the selected tab on the left menu.
	 * If set to "true", the extractor will be in autoRun mode.
	 * @name outwitHub.extractor#current
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["current"]  = false;
	   this.__defineSetter__("current", function(val) {
	   this._setAndExtractorAutoRunTest("current", val);
	   });
	   this.__defineGetter__("current", function(){
	   return this._autoRunProperties["current"];
	   });
	   */


	/**
	 * The empty value should be set to true or false depending on the checkbox "empty" of the views of each extractor.
	 * If set to "true", the extractor will clean the tree for each page loaded.
	 * If set to "true", the extractor will be in autoRun mode.
	 * @name outwitHub.extractor#empty
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["notEmpty"] = false;
	   this.__defineSetter__("empty", function(val) {
	   this._setAndExtractorAutoRunTest("notEmpty", !val);
	   });
	   this.__defineGetter__("empty", function(){
	   return !this._autoRunProperties["notEmpty"];
	   });
	   */


	/**
	 * The download value should be set to true or false depending on the checkbox "empty" of the views of each extractor.
	 * If set to "true", the extractor will be in autoRun mode. The test to download or not will be done in the bottonPanel, not here, this
	 * is used only to control the autoRun.
	 * @name outwitHub.extractor#download
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["download"] = false;
	   this.__defineSetter__("download", function(val) {
	   this._setAndExtractorAutoRunTest("download", val);
	   });
	   this.__defineGetter__("download", function() {
	   return this._autoRunProperties["download"];
	   });
	   */

	/**
	 * The export value should be set to true or false depending on the checkbox "export" of the views of each extractor.
	 * If set to "true", the extractor will be in autoRun mode. The test to export or not will be done in the bottonPanel, not here, this
	 * is used only to control the autoRun.
	 * @name outwitHub.extractor#export
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	/*
	   this._autoRunProperties["export"] = null;
	   this.__defineSetter__("export", function(val) {
	   this._setAndExtractorAutoRunTest("_export", val);
	   });
	   this.__defineGetter__("export", function() {
	   return this._autoRunProperties["export"];
	   });
	   */

	/**
	 * The done value should be set to true when the extractor have been executed.
	 * @name outwitHub.extractor#done
	 * @field
	 * @see outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @type Boolean
	 */
	this.done = (preDefAutoRunIn || false);


	/**
	 * The appendOnStop value should is set to true when the extractor put the data in the tree every time the extractor stops
	 * @name outwitHub.extractor#appendOnStop
	 * @field
	 * @type Boolean
	 */
	this.appendOnStop = appendOnStopIn;

	/**
	 * The name of the extractor
	 * @name outwitHub.extractor#name
	 * @field
	 * @type Boolean
	 */
	this.name = nameIn;

	/**
	 * The name of the viewName for the extractor
	 * @name outwitHub.extractor#viewName
	 * @field
	 * @type Boolean
	 */
	this.viewName = viewNameIn;

	/**
	 * The verbosity of the messages by event
	 * @name outwitHub.extractor#messageVerbosity
	 * @field
	 * @type Array
	 */
	this.messageVerbosity = messageVerbosityIn;

	/**
	 * The max number of elements that can be appended in the light version. undefined = all, 0 = view unavailable for light version.
	 * @name outwitHub.extractor#maxCountLightVersion
	 * @field
	 * @type Integer
	 */
	this.maxCountLightVersion = maxCountLightVersion;

	/**
	 * If the extraction of this extractor finished for this page.
	 * @name outwitHub.extractor#extractionFinished
	 * @field
	 * @type Bool
	 */
	this._extractionFinished = false;

	this.__defineGetter__("extractionFinished", function() {
		return this._extractionFinished;
	});
	this.__defineSetter__("extractionFinished", function(val) {
		if (outwitHub.extractControls.browserManager.pageLoaded && this.oneShot) {
			this.oneShot = false;
		}
		this._extractionFinished = val;
	});

	/**
	 * Test to check if an extractor hasn't been run yet and if it is autoRun
	 *
	 * @name outwitHub.extractor#shouldRunNow
	 * @function
	 * @returns {Boolean} should or not run
	 */
	this.shouldRunNow = function(evenIfNotAutorun) {
		if ((evenIfNotAutorun || this.isAutoRun()) && (!this.done)) {
			return true;
		} else {
			return false;
		}
	};


	/**
	 * Test to check if an extractor is working in the autoRun mode, which means that, it must be applied to every page loaded.
	 *
	 * @name outwitHub.extractor#isAutoRun
	 * @function
	 * @returns {Boolean} is or not in autoRun Mode
	 */
	this.isAutoRun = function() {
		//ow.log.error("isAutoRun " + this.name + " " + JSON.stringify(this._autoRunProperties));

		// XXXXXXX TO CHECK JC:
		// old: return this.preDefAutoRun || this.autoGrab || this.autoCatch
		return this.preDefAutoRun || this.autoGrab || this.autoCatch || (this.notEmpty && ow.app.module.fullId != "modules.sourcer") ||
			this.download || this.export || this.oneShot ||
			(this.smartGrab && this.current);

		/*
		//empty is the opposite because of its default value
		//if (!this.autoGrab || this._autoRunProperties["current"]) {
		  for (var prop in this._autoRunProperties) {
		    if (this._autoRunProperties[prop]) {
		      return true;
		    }
		  }
		//}
		return false;
		*/
	};

	this.toString = function() {
		var str = this.name + "(" + this.viewName + ") : ";
		for (var prop in this._autoRunProperties) {
			str += prop + "=" + this._autoRunProperties[prop] + ",  ";
		}
		return str;
	};


	/**
	 * This function is called when one property will be changed.
	 * If the extractor is NOT in the autoRun mode and it will be after the the change, we run.
	 * If the extractor is IN the autoRun mode and it not will be after the the change, we make some settings.
	 *
	 * @name outwitHub.extractor#_setAndExtractorAutoRunTest
	 * @function
	 * @param {String} property Property to be changed in the extractor
	 * @param {String | Boolean} val New value of the property
	 */
	this._setAndExtractorAutoRunTest = function(property, val) {

		//check if is in autoRun with old value
		var oState = this.isAutoRun();
		//change the property
		this._autoRunProperties[property] = val;
		//check if is in autoRun with new value
		var acState = this.isAutoRun();

		//if changed the mode
		if (oState != acState) {
			this._updateAutoRun();
		}
	};

	this._updateAutoRun = function() {
		var acState = this.isAutoRun();

		//ow.log.debug("AUTORUN ? " + acState);
		//each extractor has its own specific change...
		//one day... they will be threaten as equal... no discrimination... and the world will be as one...
		switch (this.name) {

			case "logs":
			case "history":
				break;
			case "source":
				//        case "DOMSource":
			case "scrapers":

				//if this extractor is set to autoRun, a listener is added to run the every new page loaded
				if (acState) {
					var name = String(this.name);
					this.pageLoadedListener = function(event) {
						// if simulated reload, refresh source only if source type is DOM
						if (!event.isSimulated || outwitHub.UIControls.getWidgetForView("source").sourceType == "DOM") {
							outwitHub.extractControls._showSource(name);
						}
					};

					outwitHub.extractControls.browserManager.addEventListener("pageLoaded", this.pageLoadedListener);

					//if this extractor is set NOT to autoRun, the listener is removed
				} else if (this.pageLoadedListener) {
					try {
						outwitHub.extractControls.browserManager.removeEventListener("pageLoaded", this.pageLoadedListener);
						this.pageLoadedListener = null;
					} catch (e) {}
				}
				break;

			case "links":
				break;

			default:
				//put the extractor as autoRun in the extractor manager that will run it!
				outwitHub.extractControls.extractorManager.setAutoRun(this.name, acState);
				break;
		}
	};


	/**
	 * This function is called run one extractor.
	 * This will set the property done to true. To know which extractor we are playing with, we use the name property
	 * @name outwitHub.extractor#run
	 * @function
	 */
	this.run = function(specialOptions) {
		try {
			//this is to avoid that the same extrator run more than once for each page
			this.done = true;

			//which extractor has its spefic change..
			//dont know why we stop the throbber of differents extractors here... one day...
			switch (this.name) {

				//these extractors dont do nothing
				case "logs":
				case "history":
					outwitHub.UIControls.menuTree.stopThrobber(this.name);
					break;

				case "source":
					//        case "DOMSource":
				case "scrapers":
					outwitHub.extractControls._showSource(this.name);
					break;
					//to run the scraper, we use the "source" extractor, so, we run it first
				case "scraper":
					outwitHub.extractControls._showSource("source");
					break;
				case "links":
					break;

				default:

					//if the manualRun didn't work as it should, we force the throbber
					if (!outwitHub.extractControls.extractorManager.manualRun(this.name, null, specialOptions)) {
						outwitHub.UIControls.menuTree.stopThrobber(this.name);
					}
			}

		} catch (e) {
			ow.log.error("Error running extractor " + this.name, e);
		}
	};

	this.reApply = function(specialOptions) {
		try {
			//this is to avoid that the same extrator run more than once for each page
			this.done = true;
			//which extractor has its spefic change..
			//dont know why we stop the throbber of differents extractors here... one day...
			switch (this.name) {
				// as links is not managed by us and always autoRun, stopThrobber
				case "links":
				case "images":
				case "documents":
				case "videos":
				case "audio":
				case "news":
				case "feeds":
					outwitHub.UIControls.menuTree.startThrobber(this.name);
					if (outwitHub.extractControls.browserManager.allLinksExtracted) {
						if (!outwitHub.extractControls.extractorManager.manualRun(this.name, null, specialOptions)) {
							outwitHub.UIControls.menuTree.stopThrobber(this.name);
						}
					} else {
						outwitHub.extractControls.browserManager.extractLinks();
						outwitHub.UIControls.menuTree.stopThrobber(this.name);
					}
					break;

					//these extractors dont do anything
				case "logs":
				case "history":
					outwitHub.UIControls.menuTree.stopThrobber(this.name);
					break;

				case "source":
					//        case "DOMSource":
				case "scrapers":
					outwitHub.extractControls._showSource(this.name);
					break;
					//to run the scraper, we use the "source" extractor, so, we run it first
					//case "scraper":
					//	outwitHub.extractControls._showSource("source");
				default:
					//if the manualRun didn't work as it should, we force the throbber
					if (!outwitHub.extractControls.extractorManager.manualRun(this.name, null, specialOptions)) {
						outwitHub.UIControls.menuTree.stopThrobber(this.name);
					}
			}

		} catch (e) {
			ow.log.error("Error running extractor " + this.name, e);
		}
	};
}


/**
 * Base static class to control the extraction process.
 *
 * @todo remove this awful hack for multiple sources (unique sourceViewer we move ?)
 *
 * @class
 * @static
 * @name outwitHub.extractControls
 * @memberOf outwitHub
 */
outwitHub.extractControls = {

	/**
	 * The browser Manager
	 *
	 * @type outwitKit.browserManager
	 */
	browserManager: null,

	/**
	 * The extractor manager
	 *
	 * @type outwitKit.browserExtractorManager
	 */
	extractorManager: null,

	/**
	 * A listener for starting throbber
	 *
	 * @private
	 */
	_progressStartListener: null,

	/**
	 * A listener for stopping throbber
	 *
	 * @private
	 */
	_progressStopListener: null,

	/**
	 * A listener for reinitializing extractors states
	 *
	 * @private
	 */
	_locationChangeListener: null,

	/**
	 * A list of extractors
	 * that must start with:
	 * - prefAutoRun: the extractor result is automatically catched
	 * - appendOnStop: the extractor put data in the tree in the stops or in the end
	 * - name: the name of the extractor itself
	 */
	extractors: {
		//prefAutoRun, appendOnStop , name
		links: new extractor(true, true, "links", "links", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		images: new extractor(false, false, "images", "images", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		videos: new extractor(false, false, "videos", "videos", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 0),
		audio: new extractor(false, false, "audio", "audio", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 0),
		documents: new extractor(false, false, "documents", "documents", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 0),
		emails: new extractor(false, false, "emails", "emails", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, undefined),
		tables: new extractor(false, true, "tables", "tables", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		lists: new extractor(false, true, "lists", "lists", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		text: new extractor(false, true, "text", "text", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, undefined),
		feeds: new extractor(false, false, "feeds", "feeds", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		posts: new extractor(false, true, "posts", "posts", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		wayfinder: new extractor(false, true, "wayfinder", "wayfinder", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		guess: new extractor(false, true, "guess", "guess", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		words: new extractor(false, true, "words", "words", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 0),
		scraped: new extractor(false, false, "scraped", "scraped", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, 100),
		source: new extractor(false, false, "source", "source", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, undefined),
		//    DOMSource:         new extractor(f alse,false,"DOMSource",         "DOMSource",     {'startProcess': 0, 'abortProcess':0, 'endProcess':6}, undefined),
		scrapers: new extractor(false, false, "scrapers", "scrapers", {}, undefined),
		adjacentImages: new extractor(false, false, "adjacentImages", "images", {}, undefined),
		history: new extractor(false, false, "history", "history", {
			'startProcess': 0,
			'abortProcess': 0,
			'endProcess': 6
		}, undefined)
	},


	/**
	 * Loading stuff
	 * To be called when window is loaded
	 *
	 * @function
	 */
	load: function() {
		try {
			//just import the browser and extractor managers to be easier to use
			this.browserManager = widgets.page.browserManager;

			this.extractorManager = widgets.page.extractorManager;

			// XXX should create a sourceBrowserManager and use it everywhere (including for scraped)
			outwitHub.UIControls.getWidgetForView("source").browserManager = this.browserManager;
			//    outwitHub.UIControls.getWidgetForView("DOMSource").browserManager = this.browserManager;

			// listener request sent
			this._requestSentListener = function(event) {
				outwitHub.UIControls.menuTree.startThrobber("page");
			};

			this.browserManager.addEventListener("requestSent", this._requestSentListener);


			this._docContentLoadedListener = function(event) {
				outwitHub.UIControls.onDocLoaded(event);
			};
			this.browserManager.addEventListener("docContentLoaded", this._docContentLoadedListener);


			//listener for page loaded
			this._pageLoadedListener = (function(event) {
				//page loaded
				ow.log.debug(">>>> page loaded: " + event.URI.spec);
				outwitHub.UIControls.menuTree.stopThrobber("page");
				for (var extractorName in outwitHub.extractControls.extractors) {
					var extractor = outwitHub.extractControls.extractors[extractorName];
					if (extractor.oneShot && extractor.extractionFinished) {
						extractor.oneShot = false;
					}
				}
				document.getElementById('logPanelPage').updateUI();

				coreKit.SelectionManager.active = this.browserManager.mainDocument && !this.browserManager.mainDocument.isWorkshop();
			}).bind(this);
			this.browserManager.addEventListener("pageLoaded", this._pageLoadedListener);


			// listener for location changed
			this._locationChangedListener = function(event) {
				//location changed
				ow.log.debug(">>>> location changed: " + event.URI.spec);
				try {
					var extractor;
					var extractors = outwitHub.extractControls.extractors;

					//for each extractor in the list
					for (var extractorName in extractors) {
						try {

							extractor = extractors[extractorName];

							extractor.extractionFinished = false;
							//if the extractor wasnt run ( because it is not RunOnPageLoad )
							//reset done
							if (!extractor.isAutoRun()) {
								extractor.done = false;
							}

							// XXX should be done in browserExtractorManager's locationChanged listener... don't understand.
							//outwitHub.extractControls.extractorManager.stopInstances(extractorName);

							//prepare the view ( clean the trees and other things )
							if (!event.isSimulated) {
								outwitHub.UIControls.getWidgetForView(extractorName).pageChange();
								extractor.lastCount = 0;
							}
						} catch (error) {
							//this catch is here because there will be errors from views that doesnt have
							//the page change. Like the "source" view
							//ow.log.debug("error on trying to get extractor:"+extractorName, error);
						}
					}

					if (!event.isSimulated || outwitHub.UIControls.getWidgetForView("source").sourceType == "DOM") {
						coreKit.observerService.observe(new coreKit.message("source", coreKit.message.TYPE_NOTICE, "clear", 1, null));
					}
					if (kOW_DEBUGMODE) {
						//localized message for the user saying new page was loaded
						var pString = ">> " + coreKit.localeService.getString("newPage", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
						pString = pString.replace("#pageLoaded#", event.URI.spec);
						ow.log.debug(pString);
						if (ow.app.kernel.prefs.getBool("stepByStep")) {
							ow.app.kernel.prefs.setBool("stepByStep", confirm("Last request: " + outwitKit.BrowserManager.lastRequestURI + "\n\nLast page loaded: " + event.URI.spec + "\n\nContinue in step-by-step mode?\n(Cancel to abort redirections and requests already sent.)"));
							if (!ow.app.kernel.prefs.getBool("stepByStep")) {
								outwitKit.BrowserManager.mute = true;
								setTimeout(function() {
									delete outwitKit.BrowserManager.mute;
								}, 20000);
							}
						}
					}
					// these functions test to if the autoCatch option is on , if it is, send to catch
					try {
						outwitHub.UIControls.getWidgetForView('logs').endAppend();
					} catch (e) {}
					try {
						outwitHub.UIControls.getWidgetForView('history').endAppend();
					} catch (e) {}

					//if the scraper manager was already loaded, update the current URL
					if (outwitHub.UIControls.loadScraperManagerFlag) {
						document.getElementById("scrapersWidget").currentUrl = widgets.page.currentURI.spec;
					}

					//if the macro manager was already loaded, update the current URL
					if (outwitHub.UIControls.loadMacroManagerFlag) {
						document.getElementById("macrosWidget").currentBrowserURI = widgets.page.currentURI.spec;
					}


					if (!ow.app.kernel.prefs.getBool("dontExtractLinks")) outwitHub.UIControls.menuTree.startThrobber("links");
				} catch (e) {
					ow.log.error("Error in hub locationChange listener", e);
				}
			};

			this.browserManager.addEventListener("locationChanged", this._locationChangedListener);


			this._extractorManagerListener = {
				onExtractionStart: function(anExtractorName) {
					outwitHub.UIControls.menuTree.startThrobber(anExtractorName);

					document.getElementById("infoBar").clean(anExtractorName);
					var messageVariableDictionary = {};
					messageVariableDictionary['#pageTitle#'] = widgets.page.title;
					messageVariableDictionary['#pageURI#'] = widgets.page.currentURI.spec;
					coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "startProcess", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["startProcess"], messageVariableDictionary));
				},

				onExtractionStop: function(anExtractorName, aCount, anError) {
					outwitHub.UIControls.menuTree.stopThrobber(anExtractorName);

					var widget = outwitHub.UIControls.getWidgetForView(anExtractorName);
					//not all the widgets has endAppend
					//ow.log.debug("extractControls onExtractionStop " + anExtractorName);
					try {
						widget.endAppend();
					} catch (e) {}
					widgets.page.updateCommands(["widgetPage:StopAll"]);

					//                      outwitHub.UIControls.menuTree.stopThrobber(_extractorName);
					if (kOW_DEBUGMODE) {
						//locale the message
						var pString = coreKit.localeService.getString("onExtractorStop", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
						//replace the name of the automator in the message
						pString = pString.replace("#extractorName#", anExtractorName);
						ow.log.debug(pString);
					}
					if (outwitHub.extractControls.extractors[anExtractorName].extractionFinished) {
						outwitHub.resendEndProcessMessage(anExtractorName);
						return;
					}

					outwitHub.extractControls.extractors[anExtractorName].done = true;
					outwitHub.extractControls.extractors[anExtractorName].extractionFinished = true;

					//Test if this extractor have a limited number of rows in light version and if tis limited was overflowed. In this case I send a different event.

					var maxCount = outwitHub.extractControls.extractors[anExtractorName].maxCountLightVersion;

					var widget = outwitHub.UIControls.getWidgetForView(anExtractorName);

					var countFound;
					if (typeof(widget.countFound) != "undefined") {
						countFound = widget.countFound;
					} else if (widget.XULDatasheet) {
						countFound = widget.XULDatasheet.rowCount;
					}
					if (anExtractorName == "scraped" && aCount == 0 && !/fetcher/i.test(ow.app.module.id)) {
						/*Item was already sent in onError*/
						var messageVariableDictionary = {};
						if (anError && anError.scrapedURL) {
							messageVariableDictionary['#scrapedURL#'] = anError.scrapedURL;
						}
						coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_ERROR, "endProcess.noScraperError." + (anError && anError.complement ? anError.complement : "all"), 9, messageVariableDictionary));
					} else {
						var messageVariableDictionary = {};
						messageVariableDictionary['#pageTitle#'] = widgets.page.title;
						messageVariableDictionary['#pageURI#'] = widgets.page.currentURI ? widgets.page.currentURI.spec : "";

						// XXXXX REG LEVELS
						if (!ow.app.module.uck.ok() && maxCount != undefined && widget.XULDatasheet && widget.XULDatasheet.view.rowCount >= maxCount) {
							messageVariableDictionary['#displayedColumns#'] = widget.XULDatasheet.innerTree.columns.count;
							messageVariableDictionary['#allColumns#'] = widget.JSDatasheet.getColumns().getLength();
							widget.limited = true;
							if (maxCount != 0) {
								messageVariableDictionary['#maxCount#'] = maxCount;
								messageVariableDictionary['#count#'] = countFound;
								coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "endProcess.limited", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["endProcess"], messageVariableDictionary));
								coreKit.observerService.observe(new coreKit.message("registration", coreKit.message.TYPE_NOTICE, "limited.view", 0, {
									view: anExtractorName,
									viewName: coreKit.localeService.getString('views.' + anExtractorName + 'View', 'chrome://outwit-modules/locale/localize.properties'),
									viewItems: coreKit.localeService.getString("views." + anExtractorName + ".items", "chrome://outwit-modules/locale/localize.properties"),
									maxCount: maxCount,
									neededCount: countFound
								}));
							} else {

								if (/links|images|documents/.test(anExtractorName) && ow.app.kernel.prefs.getBool("dontExtractLinks")) {
									//alert(2+anExtractorName);
									coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "disabled", 10, null));
									//messageVariableDictionary['#componentName#'] = coreKit.localeService.getString('views.'+ anExtractorName +'View', 'chrome://outwit-modules/locale/localize.properties');
									//coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "disabled", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["disabled"], null));//messageVariableDictionary));

								} else {
									messageVariableDictionary['#componentName#'] = coreKit.localeService.getString('views.' + anExtractorName + 'View', 'chrome://outwit-modules/locale/localize.properties');
									coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "endProcess.limited", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["endProcess"], messageVariableDictionary));

								}

							}
						} else {
							widget.limited = false;

							if (/links|images|documents/.test(anExtractorName) && ow.app.kernel.prefs.getBool("dontExtractLinks")) {
								//alert(1+anExtractorName);
								coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "disabled", 10, messageVariableDictionary));
								//coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "disabled", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["disabled"], messageVariableDictionary));
							} else {
								messageVariableDictionary['#count#'] = countFound;
								coreKit.observerService.observe(new coreKit.message(anExtractorName, coreKit.message.TYPE_NOTICE, "endProcess", outwitHub.extractControls.extractors[anExtractorName].messageVerbosity["endProcess"], messageVariableDictionary));

							}

						}

					}
					outwitHub.extractControls.extractors[anExtractorName].lastCount = countFound;
				},
				onExtractionAbort: function(anExtractorName) {}
			};
			this.extractorManager.addExtractionListener(this._extractorManagerListener);


			//this listener will be called when the links extractor get some links
			// XXX TODO
			this._linksExtractedListener = (function(widgetLinks) {
				return function(event) {
					//XXX Modified in 14th December 2010 - Solving multiple messages problem - ticket #780
					var extractionFinishedBefore = outwitHub.extractControls.extractors["links"].extractionFinished;
					outwitHub.extractControls.extractors["links"].extractionFinished = false;

					var links = (event.incremental ? event.extractedLinks : event.target.links);
					for (var URL in links) {
						widgetLinks.appendData(links[URL]);
					}
					//XXX Modified in 14th December 2010 - Solving multiple messages problem - ticket #780
					outwitHub.extractControls.extractors["links"].extractionFinished = extractionFinishedBefore;
					widgetLinks.stopAppend();
					if (outwitHub.extractControls.extractors["links"].extractionFinished) {
						outwitHub.resendEndProcessMessage("links");
					}
				};
			})(outwitHub.UIControls.getWidgetForView("links"));
			this.browserManager.addEventListener("linksExtracted", this._linksExtractedListener);



			//add listeners for each of views. This will sycronize the views with the extractors
			for (var extractorName in this.extractors) {
				var currentExtractor = this.extractors[extractorName];

				switch (extractorName) {
					//these first extractors has no listeners
					case "source":
						//        case "DOMSource":
					case "logs":
					case "history":
					case "scrapers":
						break;

					case "adjacentImages":
						this.extractorManager.registerDefault(extractorName);
						currentExtractor.progressListener = {
							onStart: function() {
								outwitHub.UIControls.menuTree.startThrobber("images");
								widgets.page.updateCommands(["widgetPage:StopAll"]);
							},
							onStop: function(error, isADocument, extractedData, isRunning) {
								if (!isRunning) {
									widgets.page.updateCommands(["widgetPage:StopAll"]);
									outwitHub.UIControls.menuTree.stopThrobber("images");
								}
							},
							onProgress: function( /*aData, duplicate*/ ) {
								//outwitHub.UIControls.menuTree.stopThrobber("images", 10000);
							},
							onAbort: function() {
								outwitHub.UIControls.menuTree.stopThrobber("images");
								widgets.page.updateCommands(["widgetPage:StopAll"]);
							}
						};
						this.extractorManager.addProgressListener(extractorName, currentExtractor.progressListener, true);
						break; //break of "adjacent images" case


						//for the links extractor
					case "links":
						currentExtractor.progressListener = {
							onStart: function() {
								if (!ow.app.kernel.prefs.getBool("dontExtractLinks")) {
									outwitHub.UIControls.menuTree.startThrobber("links");
									widgets.page.updateCommands(["widgetPage:StopAll"]);
								}
							},
							onStop: function(error, isADocument, extractedData, isRunning) {
								if (!isRunning) {
									widgets.page.updateCommands(["widgetPage:StopAll"]);
									outwitHub.UIControls.menuTree.stopThrobber("links");
								}
							},
							onProgress: function(aData) {
								//outwitHub.UIControls.menuTree.stopThrobber("links", 10000);
							},
							onAbort: function() {
								outwitHub.UIControls.menuTree.stopThrobber("links");
								widgets.page.updateCommands(["widgetPage:StopAll"]);
							}
						};
						if (this.browserManager.linkInstanceManager) this.browserManager.linkInstanceManager.addProgressListener(currentExtractor.progressListener, true, true);
						break; //break of "links" case

						//the default listeners are divided in parts:
						// view that append data onStop and views that append data onProgress
						// XXX : dont know why we use onStop/onProgress for each of then
					default:

						// XXX Special case for scraped, because it uses the source view
						if (extractorName == "scraped") {
							var manualRunMethod, autoRunMethod;
							(function(sourceView) {

								manualRunMethod = function(browserManager, extractorInstance, options) {
									var URI = outwitHub.extractControls.browserManager.currentURI;
									//ow.log.error("manualRunMethod ---- options.scraper ");
									//ow.log.error("manualRunMethod ---- options.scraper " + (options.scraper ? options.scraper.name + " " + options.scraper.getData() : "" ));
									var scraper = options.scraper || ow.automation.util.getBestAutomatorByTypeForURL(URI.spec, "scraper");

									//ow.log.error("URI: " + browserManager.mainDocument.URI);
									//ow.log.error("request URI " + browserManager.mainDocument.requestURI ? browserManager.mainDocument.requestURI : "");
									if (browserManager.mainDocument.URI && browserManager.redirectedRequests[browserManager.mainDocument.URI.spec]) {
										options.requestedURI = outwitKit.URLHistoryService.getNsIURIFromString(browserManager.redirectedRequests[browserManager.mainDocument.URI.spec].requestURI);
									}
									if (!scraper) {
										return 0;
									}
									//ow.log.error("manualRunMethod ---- X");
									var UIHelper = ow.ui.automation.ManagerHelper.create(mainKit.automatorManagerService.getTypedManager("scraper"));
									UIHelper.decryptAutomator(scraper, false, function(scraper) {
										outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape = false;
										switch (scraper.getOption("source")) {
											case "allFrames":
												if (widgets.page.setStartTime() || theScraperObject.isDypto()) {
													var getSource = function() {
														return outwitKit.stringHelpers.concatFrameSources();
													};
													extractorInstance.fromString(getSource, URI, scraper, options);
												}
												break;
											case "DOM":
												var getSource = function() {
													return browserManager.mainDocument.getCompleteSource();
												};
												extractorInstance.fromString(getSource, URI, scraper, options);
												break;
											default:
												//ow.log.error("manualRunMethod ---- Z");
												outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape = false;
												var getSource = function() {
													return browserManager.mainDocument.responseSource;
												};
												extractorInstance.fromString(getSource, URI, scraper, options);
										}
									});
									return 1;
								};

								autoRunMethod = function(event, extractorInstance, options) {
									var browserManager = event.target;
									var URI = browserManager.currentURI;
									//ow.log.error("autoRunMethod ---- options.scraper: " + (options.scraper ? options.scraper.name + " " + options.scraper.getData() : "" ));

									var scraper;
									if (outwitKit.macroExecutionManager.isRunningMacro() && outwitKit.macroExecutionManager.selectedScraper) {
										//ow.log.error("autoRunMethod ---- 0 "+[URI.spec, outwitKit.macroExecutionManager.selectedScraper]);
										scraper = ow.automation.util.getBestAutomatorForURL(URI.spec, [outwitKit.macroExecutionManager.selectedScraper], true);
									} else if (URI) {
										//ow.log.error("autoRunMethod ---- 1a:"+URI.spec);
										scraper = ow.automation.util.getBestAutomatorByTypeForURL(URI.spec, "scraper");
										//ow.log.error("autoRunMethod ---- 1b:"+scraper);
									}
									//ow.log.error("autoRunMethod ---- 2");
									if (!scraper) {
										return 0;
									}
									//ow.log.error("autoRunMethod ---- 3");
									//ow.log.error("browserManager.mainDocument: " + browserManager.mainDocument);
									//ow.log.error(">>> browserManager.mainDocument.URI: " + JSON.stringify(browserManager.mainDocument.URI));
									//ow.log.error(">>> auto URI: " + browserManager.mainDocument.URI.spec);
									//ow.log.error(">>> request URI " + browserManager.mainDocument.requestURI ? browserManager.mainDocument.requestURI : "");
									//              if (browserManager.mainDocument.URI && browserManager.redirectedRequests[browserManager.mainDocument.URI.spec]) {
									//                try{
									//                  options.requestedURI = outwitKit.URLHistoryService.getNsIURIFromString(browserManager.redirectedRequests[browserManager.mainDocument.URI.spec].requestURI);
									//                } catch(e){
									//                  ow.log.error(e+"\nline:"+e.lineNumber);
									//                  alert(e+"\nline:"+e.lineNumber);
									//                }
									//              } else {
									//				  ow.log.warning("autoRunMethod ---- 3b:\n" + browserManager.mainDocument.URI ?
									//				  	(browserManager.mainDocument.URI + "\n" + browserManager.redirectedRequests[browserManager.mainDocument.URI.spec]):
									//					"no main document uri");
									//              }


									options.requestedURI = browserManager.mainDocument ? browserManager.mainDocument.requestURI : null;

									ow.log.debug("AUTORUN wasFromScroll ? " + event.wasFromScroll, true);
									if (event.wasFromScroll) options.disableScroll = true;

									//ow.log.error("autoRunMethod ---- 4");
									var UIHelper = ow.ui.automation.ManagerHelper.create(mainKit.automatorManagerService.getTypedManager("scraper"));
									UIHelper.decryptAutomator(scraper, false, function(scraper) {
										//ow.log.error("autoRunMethod ---- A");
										if (scraper) {
											switch (scraper.getOption("source")) {
												case "allFrames":
													if (widgets.page.setStartTime() || theScraperObject.isDypto()) {
														outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape = false;
														//XXX Added on 9th December 2010 - For problem on http://livingsocial.com/deals/[424:50000] Drupal Ticket: 2873
														//ow.log.error("autoRunMethod ---- Ba");
														if (outwitHub.extractControls.browserManager.nLoaded == 1) {
															var getSource = function() {
																return outwitKit.stringHelpers.concatFrameSources();
															};
															// XXX Hack, to prevent autopilot to go to next page before
															extractorInstance._runningCount++;
															window.setTimeout(function() {
																browserManager.mainDocument.updateCompleteSource();
																extractorInstance._runningCount--;
																//ow.log.error("autoRunMethod ---- Bb");
																extractorInstance.fromString(getSource, URI, scraper, options);
															}, Math.min(ow.app.kernel.prefs.getInt("browse.tempo.max") / 3, 3) * 1000);

															break;
														}
													}
													case "DOM":
														outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape = false;
														//XXX Added on 9th December 2010 - For problem on http://livingsocial.com/deals/[424:50000] Drupal Ticket: 2873
														//ow.log.error("autoRunMethod ---- Ba");
														if (outwitHub.extractControls.browserManager.nLoaded == 1) {
															// XXX Hack, to prevent autopilot to go to next page before
															extractorInstance._runningCount++;
															var getSource = function() {
																return browserManager.mainDocument.getCompleteSource();
															};
															window.setTimeout(function() {
																browserManager.mainDocument.updateCompleteSource();
																extractorInstance._runningCount--;
																//ow.log.error("autoRunMethod ---- Bb");
																extractorInstance.fromString(getSource, URI, scraper, options);
															}, Math.min(ow.app.kernel.prefs.getInt("browse.tempo.max") / 3, 3) * 1000);
														}
														break;
													default:
														//ow.log.error("autoRunMethod ---- Ca ---- nLoaded="+outwitHub.extractControls.browserManager.nLoaded );
														outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape = false;
														//XXX Added on 9th December 2010 - For problem on http://livingsocial.com/deals/[424:50000] Drupal Ticket: 2873
														if (outwitHub.extractControls.browserManager.nLoaded == 1) {
															var getSource = function() {
																return browserManager.mainDocument.responseSource;
															};
															//ow.log.error("autoRunMethod ---- Cb");
															extractorInstance.fromString(getSource, URI, scraper, options);
														}
											}
										}
									});
									return 1;
								};

							})(outwitHub.UIControls.getWidgetForView("source"));

							this.extractorManager.register(
								"scraped", outwitKit.scrapeExtractor,
								"browserEvent", "pageLoaded",
								autoRunMethod, manualRunMethod);
						} else {
							this.extractorManager.registerDefault(extractorName);
						}

						currentExtractor.progressListener = (function(widget, _extractorName) {
							if (currentExtractor.appendOnStop) {
								//we are buildig a listener here... we just return it to the addListener function
								return {
									onStart: function() {
										//locale the starting to run extractor message
										if (kOW_DEBUGMODE) {
											var pString = coreKit.localeService.getString("runExtractor", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
											//replace the name of the automator in the message
											pString = pString.replace("#extractorName#", _extractorName);
											ow.log.debug("pString (A): " + pString);
										}
										outwitHub.UIControls.menuTree.startThrobber(_extractorName);

										widgets.page.updateCommands(["widgetPage:StopAll"]);
										try {
											widget.startAppend();
										} catch (e) {}
									},
									onError: function(error) {
										//alert(JSON.stringify(error));
										if (error && error.code != coreKit.owError.NO_ERROR) {
											if (error.code == coreKit.owError.NO_SCRAPER_ERROR && !/fetcher/i.test(ow.app.module.id)) {
												var messageVariableDictionary = {};
												messageVariableDictionary['#scrapedURL#'] = error.scrapedURL;
												coreKit.observerService.observe(new coreKit.message(_extractorName, coreKit.message.TYPE_ERROR, "extractionError.noScraperError.item", 9, messageVariableDictionary));
											} else {
												// XXX
												if (_extractorName != "emails") {
													var messageVariableDictionary = {};
													messageVariableDictionary['#errorMessage#'] = coreKit.localeService.getString(error.code, 'chrome://outwit-kernel/locale/messages/errors.properties');
													messageVariableDictionary['#errorStack#'] = error.stack;
													messageVariableDictionary['#errorCode#'] = error.code;
													messageVariableDictionary['#componentName#'] = ow.data.normalization.normalize(_extractorName, "n", true);
													coreKit.observerService.observe(new coreKit.message(_extractorName, coreKit.message.TYPE_ERROR, "extractionError", 9, messageVariableDictionary));
												}
											}
										}
									},
									onStop: function(error, isADocument, extractedData, isRunning) {
										if (extractedData) {
											for (var i = 0; i < extractedData.length; ++i) {
												widget.appendData(extractedData[i]);
											}
										}
										try {
											widget.stopAppend();
										} catch (e) {}

										if (error && error.code == coreKit.owError.NO_SCRAPER_ERROR && !/fetcher/i.test(ow.app.module.id)) {

											var messageVariableDictionary = {};
											if (error.scrapedURL) {
												messageVariableDictionary['#scrapedURL#'] = error.scrapedURL;
											}
											coreKit.observerService.observe(new coreKit.message(_extractorName, coreKit.message.TYPE_ERROR, "endProcess.noScraperError." + error.complement, 9, messageVariableDictionary));
										}
									},
									onProgress: function( /*aData*/ ) {
										//do nothing here
									},
									onAbort: function() {
										outwitHub.UIControls.menuTree.stopThrobber(_extractorName);
										widgets.page.updateCommands(["widgetPage:StopAll"]);
									}
								};

							} else {
								//view with append data onProgress

								//we are buildig a listener here... we just return it to the addListener function
								return {
									_endOnStop: false,
									_firstStart: true,
									onStart: function() {
										if (this._firstStart) {
											this._firstStart = false;

											//					ow.log.error(">> ####### testing min delay ########");
											//					if(outwitKit.currentExploration && outwitKit.currentExploration.minDelay) {
											//						var elapsed = Date.now() - (outwitKit.currentExploration.previousScrapeTime||0);
											//						ow.log.notice(">> ####### min delay: "+outwitKit.currentExploration.minDelay+", elapsed: "+elapsed+" ########");
											//						if(elapsed < outwitKit.currentExploration.minDelay) {
											//							ow.log.notice(">> ####### CANCELLING THIS SCRAPE [0]: min delay is "+outwitKit.currentExploration.minDelay+" ########");
											//							return 0;
											//						}
											//						outwitKit.currentExploration.previousScrapeTime = Date.now();
											//					}

											if (kOW_DEBUGMODE) {
												//locale the starting to run extractor message
												var pString = coreKit.localeService.getString("runExtractor", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
												//replace the name of the automator in the message
												pString = pString.replace("#extractorName#", _extractorName);
												ow.log.debug("pString (B): " + pString);
											}
											widgets.page.updateCommands(["widgetPage:StopAll"]);
											outwitHub.UIControls.menuTree.startThrobber(_extractorName);
											try {
												widget.startAppend();
											} catch (e) {}
											if (outwitHub.extractControls.browserManager.allLinksExtracted) {
												this._endOnStop = true;
											}
										}
									},
									onStop: function(error, isADocument, extractedData, isRunning) {
										/*if (_extractorName == "images") {
										  ow.log.debug("IMAGES onStop " + _extractorName +  " " + extractedData.length + "\n");
										  }*/

										try {
											widget.stopAppend();
										} catch (e) {}
										if (!isRunning) {
											if (this._endOnStop || outwitHub.extractControls.browserManager.allLinksExtracted || error) {
												this._firstStart = true;

												if ((_extractorName == "scraped") && (outwitHub.UIControls.scrapedAutoHackWentToScrapedBecauseOfFastScrape)) {
													widget.endAppend();
												}

												outwitHub.UIControls.menuTree.stopThrobber(_extractorName);
												this._endOnStop = false;
												if (kOW_DEBUGMODE) {
													//locale the message
													// ow.log.debug("\n\nNOT apend on stop STOP: "+extractorName);
													//locale the message
													var pString = coreKit.localeService.getString("onExtractorStop", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
													//replace the name of the automator in the message
													pString = pString.replace("#extractorName#", _extractorName);
													ow.log.debug("pString (C): " + pString);
												}
											}
											widgets.page.updateCommands(["widgetPage:StopAll"]);
										}
									},
									onError: function(error) {
										// alert(JSON.stringify(error));
										if (error && error.code != coreKit.owError.NO_ERROR && !/fetcher/i.test(ow.app.module.id)) {
											if (error.code == coreKit.owError.NO_SCRAPER_ERROR) {
												var messageVariableDictionary = {};
												messageVariableDictionary['#scrapedURL#'] = error.scrapedURL;
												coreKit.observerService.observe(new coreKit.message(_extractorName, coreKit.message.TYPE_ERROR, "extractionError.noScraperError.item", 9, messageVariableDictionary));
											} else {
												// XXX
												if (_extractorName != "emails") {
													var messageVariableDictionary = {};
													messageVariableDictionary['#errorMessage#'] = coreKit.localeService.getString(error.code, 'chrome://outwit-kernel/locale/messages/errors.properties');
													messageVariableDictionary['#errorStack#'] = error.stack;
													messageVariableDictionary['#errorCode#'] = error.code;
													messageVariableDictionary['#componentName#'] = ow.data.normalization.normalize(_extractorName, "n", true);
													//(aComponent, aType, aEvent, aSeverity, aVariableDictionary, aDate)
													coreKit.observerService.observe(new coreKit.message(_extractorName, coreKit.message.TYPE_ERROR, "extractionError", 9, messageVariableDictionary));
												}
											}

										}
									},
									onProgress: function(aData) {
										//ow.log.debug("## onProgress APPENDING DATA"+ JSON.stringify(aData));
										widget.appendData(aData);
									},
									onAbort: function() {
										outwitHub.UIControls.menuTree.stopThrobber(_extractorName);
										widgets.page.updateCommands(["widgetPage:StopAll"]);
									}
								};
							}
						})(outwitHub.UIControls.getWidgetForView(extractorName), extractorName);


						this.extractorManager.addProgressListener(extractorName, currentExtractor.progressListener, true);

						if (currentExtractor.isAutoRun()) {
							//ow.log.debug("SETTING AUTORUN: "+extractorName);
							this.extractorManager.setAutoRun(extractorName, true);
						}
						break;
				}


				/*

				   if (extractorName == "links") {
				   currentExtractor._allLinksExtractedListener = (function(widget, _extractorName) {
				   return function(evt) {
				   if (outwitHub.extractControls.extractors[_extractorName].isAutoRun()) {
				   try { widget.endAppend(); } catch(e) {}
				   widgets.page.updateCommands(["widgetPage:StopAll"]);
				   outwitHub.UIControls.menuTree.stopThrobber(_extractorName);
				   }
				   };
				   })(outwitHub.UIControls.getWidgetForView(extractorName), extractorName);

				   this.browserManager.addEventListener("allLinksExtracted", currentExtractor._allLinksExtractedListener);
				   }
				   */

				//infobar messages
				/*      currentExtractor._allStoppedMessageListener = (function(widget, extractorName) {
				        return function(evt) {
				        };
				        })(outwitHub.UIControls.getWidgetForView(extractorName), extractorName);
				        this.browserManager.addEventListener("allStopped", currentExtractor._allStoppedMessageListener);*/

				var widget = outwitHub.UIControls.getWidgetForView(extractorName);
				if (widget) {
					outwitHub.UIControls.menuTree.updateTreeView(widget);
				}
			}

			if (ow.app.module.fullId != "modules.sourcer") {
				this.reloadImagesAutoRunFromPref = function() {
					this.extractors["images"].preDefAutoRun = !ow.app.kernel.prefs.getBool("images.ondemandonly");
				};
				ow.app.kernel.prefs.addObserver("images.ondemandonly", this.reloadImagesAutoRunFromPref, this);
				this.reloadImagesAutoRunFromPref();
			}
		} catch (e) {
			ow.log.error("Error loading extractor in line " + e.lineNumber, e);
		}
	},


	/**
	 * Unloading stuff properly
	 * To be called when window is unloaded
	 *
	 * @function
	 */
	unload: function() {
		if (ow.app.module.fullId != "modules.sourcer") {
			ow.app.kernel.prefs.removeObserver("images.ondemandonly", this.reloadImagesAutoRunFromPref, this);
		}

		this.extractorManager.removeExtractionListener(this._extractorManagerListener);

		this.browserManager.removeEventListener("linksExtracted", this._linksExtractedListener);
		this.browserManager.removeEventListener("pageLoaded", this._docContentLoadedListener);
		this.browserManager.removeEventListener("requestSent", this._requestSentListener);
		this.browserManager.removeEventListener("pageLoaded", this._pageLoadedListener);
		this.browserManager.removeEventListener("locationChanged", this._locationChangedListener);
		for (var extractorName in this.extractors) {
			switch (extractorName) {
				case "source":
				case "logs":
				case "history":
				case "scrapers":
					break;
				case "links":
					if (this.browserManager.linkInstanceManager) this.browserManager.linkInstanceManager.removeProgressListener(this.extractors[extractorName].progressListener, true);
					break;
				default:
					this.extractorManager.removeProgressListener(extractorName, this.extractors[extractorName].progressListener);
					break;
			}
			if (this.extractors[extractorName].pageLoadedListener) {
				try {
					this.browserManager.removeEventListener("pageLoaded", this.extractors[extractorName].pageLoadedListener);
				} catch (e) {}
			}

		}
	},

	/**
	 * Super hack plus plus !!!
	 * This function calls the "run" of the "source" extractor. This is called when the user click on the scraper, source or scraped
	 *
	 * @name outwitHub.extractControls#_showSource
	 * @function
	 * @param {String} aView A name of a view that contains the "source"
	 */
	_showSource: function(aView) {
		ow.log.warning("_showSource called " + aView);
		/*
		//get the current view
		var view = outwitHub.UIControls.getWidgetForView((aView == "scrapers" ? "source" : aView));
		//load the view in the colorized way
		view.loadSource(true);
		//stop the throbber
		//*/
		document.getElementById("scrapersSourceWidget").loadSource(true);
		outwitHub.UIControls.menuTree.stopThrobber(aView);
	},

	/**
	 * Set an extractor as the current one, enabling it if not already autoRun.
	 *
	 * @function
	 * @param {Strin} anExtractorName
	 */
	setCurrent: function(anExtractorName, dontRunExtractor) {
		if (kOW_DEBUGMODE) {
			//locale the message of setting current view
			var pString = coreKit.localeService.getString("setCurrent", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
			pString = pString.replace("#extractorName#", anExtractorName);
			ow.log.notice(pString + " extractControls>setCurrent");
		}
		var extractorName;
		try {
			for (extractorName in this.extractors) {
				var extractor = this.extractors[extractorName];

				if (extractorName == anExtractorName) {
					if (!extractor.current) {
						//alert(extractorName+'\n'+Object.keys(this.extractors));
						extractor.current = true;
						if (extractor.shouldRunNow()) {
							outwitHub.UIControls.menuTree.startThrobber(extractorName);
							//window.setTimeout(function() {
							// XXX and if we are currently loading the page ?
							// => Simple timeout? Or more complicated (but need to store which
							// browser load events have been fired for each doc...)
							if (!dontRunExtractor) {
								var timerId = coreKit.timerService.setTimer(100, null, function(_extractorName) {
									outwitHub.extractControls.extractors[_extractorName].run();
								}, null, extractorName);
								//ow.log.debug("&&&&&&&&&&&&&&&&& SETCURRENT SETTING TIMER " + timerId);
							}
						}
					}

				} else {
					//ow.log.debug("setCurrent: extractors[\"" + extractorName + "\"].current = false (autoCatch=" + extractor.autoCatch + ", autoRun=" + extractor.preDefAutoRun + ", done=" + extractor.done + ")\n");
					//ow.log.debug("extractor.viewName=" + extractor.viewName + " --  outwitHub.UIControls.getWidgetForView(extractor.viewName)=" + outwitHub.UIControls.getWidgetForView(extractor.viewName));
					extractor.current = false;
					outwitHub.UIControls.getWidgetForView(extractor.viewName).removeAttribute("current");
				}
			}

			try {
				if (this.extractors[anExtractorName]) {
					outwitHub.UIControls.getWidgetForView(this.extractors[anExtractorName].viewName).setAttribute("current", "true");
				}
			} catch (e) {
				ow.log.error("Error " + e + " - Extractor:" + this.extractors[anExtractorName], e);
			}

		} catch (e) {
			ow.log.error("4 Error setting current extractor to " + extractorName + " line " + e.lineNumber, e);
		}
	},

	reApply: function(anExtractorName, force) {
		// XXX TODO RELOAD SOURCE FOR SOURCE-BASED EXTRACTORS (should be detected and done in source getter)
		// XXX still valid?

		if (!anExtractorName) {
			anExtractorName = outwitHub.UIControls.menuTree.getSelectedView();
		}
		var extractor = outwitHub.extractControls.extractors[anExtractorName];
		extractor.done = false;
		var view = outwitHub.UIControls.getWidgetForView(anExtractorName);
		//ow.log.error(anExtractorName + " "+ view + " " + view.bottomPanel);
		var options = view.bottomPanel.optionsPanel.options;
		for (var optionName in options) {
			var newValue = view.bottomPanel.optionsPanel.getOptionValue(optionName);
			if (options[optionName].type == "option" && outwitHub.extractControls.extractorManager.getOptionValue(anExtractorName, optionName) != newValue) {
				outwitHub.extractControls.extractorManager.setOptionValue(anExtractorName, optionName, newValue);
			}
		}
		extractor.oneShot = true;
		if (extractor.shouldRunNow(true) || force) {
			outwitHub.UIControls.menuTree.startThrobber(anExtractorName);
			view.pageChange();
			extractor.reApply();
		}
	},

	// ==================================================================================================================
	// ================================================== FAST SCRAPE =================================================
	// ==================================================================================================================

  checkIfCatch: function(widget){
    var catchEvery, datasheet, datasheetLength;
    widget = outwitHub.UIControls.getWidgetForView(widget);
    datasheet = widget.JSDatasheet;
    datasheetLength = datasheet.getLength();
    //ow.log.debug("checkIfCatch -- catchEvery = "+ow.app.kernel.prefs.getInt("catchEvery"));
    if (catchEvery = ow.app.kernel.prefs.getInt("catchEvery")) {
      //ow.log.debug("checkIfCatch -- 1 = ");
      if (catchEvery && datasheetLength >= catchEvery) {
        //ow.log.debug("checkIfCatch -- 2 = ");
        datasheet.selection.selectAll();
        widget.sendToCatch();
        datasheet.deleteSelectedRows();
      }
    }
  },


	// ===================== SCRAPE STRINGS =====================



	scrapeStrings: function(stringsToScrape, aScraper, options) {
		//ow.log.error("############ 1:"+JSON.stringify(options));
		if (!ow.app.module.uck.uk()) {
			alert("This feature is only available in the Ultimate edition");
			return false;
		}
		coreKit.timerService.wait(500);

		widgets.page.browserPilot.fastRunning = true;
		widgets.page.browserPilot.lastExplorationStart = Date.now();
		ow.ui.status.updateInfoBox();

		if (!outwitKit.currentExploration || !outwitKit.currentExploration.startTime) {
			outwitKit.currentExploration = {
				visitedPageCount: 0,
				startTime: Date.now(),
				scrapeCount: 0,
				stacks: {},
				explorationType: "auto.fast",
			};
		}

		if (kOW_DEBUGMODE) {
			//locale the message
			var pString = coreKit.localeService.getString("applyingScrapersStart", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
			//replace the name of the automator in the message
			pString = pString.replace("#numberOfLines#", stringsToScrape.length);
			ow.log.debug(pString);
		}
		var doApply = function(scraper) {
			var forceScraperList = (scraper && !Array.isArray(scraper) ? [scraper] : scraper);
			var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
			if (ow.app.kernel.prefs.getBool("fastScrape.selectScrapedView")) outwitHub.UIControls.selectScrapedView(true);
			scrapedWidget.pageChange();
			// scrapedWidget.JSDatasheet.clearSort();

			var extractorInstance = outwitHub.extractControls.extractorManager.getInstanceForLocation("scraped", "scrapeStrings");

			var progressListener = {
				_wasAborted: false,
				onStart: function() {
					//ow.log.error("------ onStart:"+(new Error).stack);
					ow.app.kernel.prefs.setBool("sound.paused", true);
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.noAutoResize = true;
					this._wasAborted = false;
					var messageVariableDictionary = {};
					coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "startProcess", 6, messageVariableDictionary));
					outwitHub.extractControls.extractors["scraped"].extractionFinished = false;
				},
				onStop: function(error) {
					//ow.log.error("Date.now() - widgets.page.browserPilot.lastExplorationEnd: "+(Date.now() - widgets.page.browserPilot.lastExplorationEnd));
					//ow.log.error("------ onStop:"+error+" --- "+ (new Error).stack);
					ow.app.kernel.prefs.setBool("sound.paused", false);
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.autoResize();
					scrapedWidget.XULDatasheet.noAutoResize = false;

					try {
						if (!this._wasAborted && !outwitHub.extractControls.extractors["scraped"].extractionFinished) {
							outwitHub.extractControls.extractors["scraped"].extractionFinished = true;
							var eventName = "";
							var messageVariableDictionary = {};
							var maxCount = outwitHub.extractControls.extractors['scraped'].maxCountLightVersion;

							if (error && error.code == coreKit.owError.NO_SCRAPER_ERROR && !/fetcher/i.test(ow.app.module.id)) {
								var messageVariableDictionary = {};
								if (error.scrapedURL) {
									messageVariableDictionary['#scrapedURL#'] = error.scrapedURL;
								}
								coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_ERROR, "endProcess.noScraperError." + error.complement, 9, messageVariableDictionary));
							} else {
								messageVariableDictionary['#displayedColumns#'] = scrapedWidget.XULDatasheet.innerTree.columns.count;
								messageVariableDictionary['#allColumns#'] = scrapedWidget.JSDatasheet.getColumns().getLength();
								if (!ow.app.module.uck.ok() && maxCount != undefined && outwitHub.UIControls.getWidgetForView("scraped").XULDatasheet.view.rowCount >= maxCount) {} else {
									messageVariableDictionary['#count#'] = scrapedWidget.XULDatasheet.view.rowCount;
									coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_NOTICE, "endProcess.scrapeStrings", 6, messageVariableDictionary));
									coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "endProcess", 9, messageVariableDictionary));
								}
							}
						}
						outwitHub.extractControls.extractorManager.stopInstance("scraped", "scrapeStrings");
            outwitHub.extractControls.checkIfCatch("scraped");
					} catch (e) {
						ow.log.error("Scrape onStop error " + e);
						//alert("Scrape onStop error " + e);
					}
					//ow.log.warning("#5 changing widgets.page.browserPilot.fastRunning from "+ widgets.page.browserPilot.fastRunning + " to false")

					if (widgets.page.browserPilot.fastRunning) {
						widgets.page.browserPilot.fastRunning = false;
						widgets.page.browserPilot.lastExplorationEnd = Date.now();
						ow.ui.status.updateInfoBox();
						//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration");
						delete outwitKit.currentExploration;
						widgets.page.resetStacks();
					}
				},
				onProgress: function() {
					//ow.log.error("------ onProgress:"+ (new Error).stack);
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					coreKit.SoundService.play(coreKit.SoundService.SND_TICK);
					var len = scrapedWidget.JSDatasheet.getLength();
					if (len <= 10 || len % 50 <= 2) {
						scrapedWidget.XULDatasheet.autoResize();
					}
          outwitHub.extractControls.checkIfCatch("scraped");
				},
				onAbort: function() {
					ow.app.kernel.prefs.setBool("sound.paused", false);
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.autoResize();
					scrapedWidget.XULDatasheet.noAutoResize = false;
					this._wasAborted = true;
					var messageVariableDictionary = {};
					coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "abortProcess", 9, messageVariableDictionary));
					// XXX very very very bad
					outwitHub.extractControls.extractors["scraped"].extractionFinished = true;
					delete outwitKit.currentExploration;
					widgets.page.browserPilot.fastRunning = false;
					widgets.page.browserPilot.lastExplorationEnd = Date.now();
					ow.ui.status.updateInfoBox();
					//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration");
					widgets.page.resetStacks();
				},
				onError: function() {

				}
			};

			extractorInstance.addProgressListener(progressListener);

			outwitHub.extractControls.extractors["scraped"].done = true;
			// XXX JC
			//ow.log.error("############ 2:"+JSON.stringify(options));
			var mergedOptions = ow.util.js.copy(widgets.page.extractorManager._registered["scraped"].options);
			if (options) {
				Object.keys(options).forEach(function(optName) {
					mergedOptions[optName] = options[optName];
				});
			}
			//ow.log.error("############ 3:"+JSON.stringify(mergedOptions));
			//outwitKit.scrapeExtractor.prototype.fromStrings = function(aStringArray, aSourceURI, aScraper, options) {
			extractorInstance.fromStrings(stringsToScrape, {
				spec: "local string",
				host: "local query directory"
			}, forceScraperList[0], mergedOptions);
			if (kOW_DEBUGMODE) {
				//locale the message
				var pString = coreKit.localeService.getString("applyingScrapersStop", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
				//replace the name of the automator in the message
				pString = pString.replace("#numberOfLines#", stringsToScrape.length);
				ow.log.debug(pString);
			}
		};

		var forceScraperList;
		if (typeof(aScraper) == "number") {
			var scraper = mainKit.automatorManagerService.getTypedManager("scraper").get(aScraper);
			forceScraperList = [scraper];
		} else if (ow.automation.Automator.isPrototypeOf(aScraper)) {
			forceScraperList = [aScraper];
		} else if (Array.isArray(aScraper)) {
			forceScraperList = aScraper;
		} else {
			forceScraperList = false;
		}

		if (forceScraperList) {
			var UIHelper = ow.ui.automation.ManagerHelper.create(mainKit.automatorManagerService.getTypedManager("scraper"));
			UIHelper.decryptAutomator(forceScraperList, false, doApply);
		} else {
			doApply(forceScraperList);
		}
	},





	// ===================== FAST SCRAPE =====================





	fastScrape: function(aURIList, aScraper, contextualData, options) {
		//ow.log.warning("#4 changing widgets.page.browserPilot.fastRunning from " + widgets.page.browserPilot.fastRunning + "to true");
		//ow.log.warning((new Error).stack);
		coreKit.timerService.wait(500);

		widgets.page.browserPilot.fastRunning = true;
		widgets.page.browserPilot.lastExplorationStart = Date.now();
		ow.ui.status.updateInfoBox();
		// XXXXX JUST FOR TESTING
		// widgets.page.browserPilot.scope = prompt("Enter options.scope","site within")

		if (!outwitKit.currentExploration || !outwitKit.currentExploration.startTime) {
			outwitKit.currentExploration = {
				visitedPageCount: 0,
				startTime: Date.now(),
				scrapeCount: 0,
				stacks: {},
				explorationType: "auto.fast",
			};
		}

		//xxx jc
		// from beans to nsIURIs
		var URIList = aURIList.map(function(elt) {
			try {
				return elt.URI;
			} catch (e) {
				return null;
			}
		});
		// XXXX is this right? what about lists of URLs?
		//outwitKit.currentExploration.referenceURI = URIList[0];
		if (kOW_DEBUGMODE) {
			//locale the message
			var pString = coreKit.localeService.getString("applyingScrapersStart", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
			//replace the name of the automator in the message
			pString = pString.replace("#numberOfLines#", URIList.length);
			ow.log.debug(pString);
		}

		var doApply = function(scraper) {
			var forceScraperList = (scraper && !Array.isArray(scraper) ? [scraper] : scraper);
			var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
			if (ow.app.kernel.prefs.getBool("fastScrape.selectScrapedView")) outwitHub.UIControls.selectScrapedView(true);
			scrapedWidget.pageChange();
			// scrapedWidget.JSDatasheet.clearSort();

			var extractorInstance = outwitHub.extractControls.extractorManager.getInstanceForLocation("scraped", "fastScrape");

			var progressListener = {
				_wasAborted: false,
				onStart: function() {
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.noAutoResize = true;
					this._wasAborted = false;
					var messageVariableDictionary = {};
					coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "startProcess", 6, messageVariableDictionary));
					outwitHub.extractControls.extractors["scraped"].extractionFinished = false;
				},
				onStop: function(error) {
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.noAutoResize = false;
					scrapedWidget.XULDatasheet.autoResize();

					try {
						if (!this._wasAborted && !outwitHub.extractControls.extractors["scraped"].extractionFinished) {
							var scrapedWidget = outwitHub.UIControls.getWidgetForView('scraped');
							var eventName = "";
							var messageVariableDictionary = {};
							var maxCount = outwitHub.extractControls.extractors['scraped'].maxCountLightVersion;

							if (error && error.code == coreKit.owError.NO_SCRAPER_ERROR && !/fetcher/i.test(ow.app.module.id)) {
								var messageVariableDictionary = {};
								if (error.scrapedURL) {
									messageVariableDictionary['#scrapedURL#'] = error.scrapedURL;
								}
								coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_ERROR, "endProcess.noScraperError." + error.complement, 9, messageVariableDictionary));
							} else {
								messageVariableDictionary['#displayedColumns#'] = scrapedWidget.XULDatasheet.innerTree.columns.count;
								messageVariableDictionary['#allColumns#'] = scrapedWidget.JSDatasheet.getColumns().getLength();
								if (!ow.app.module.uck.ok() && maxCount != undefined && outwitHub.UIControls.getWidgetForView("scraped").XULDatasheet.view.rowCount >= maxCount) {
									messageVariableDictionary['#maxCount#'] = maxCount;
									messageVariableDictionary['#count#'] = scrapedWidget.countFound;
									coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_NOTICE, "endProcess.fastScrape.limited", 6, messageVariableDictionary));
									coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "endProcess.limited", 9, messageVariableDictionary));
								} else {
									messageVariableDictionary['#count#'] = scrapedWidget.XULDatasheet.view.rowCount;
									coreKit.observerService.observe(new coreKit.message("scraped", coreKit.message.TYPE_NOTICE, "endProcess.fastScrape", 6, messageVariableDictionary));
									coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "endProcess", 9, messageVariableDictionary));
								}
							}
							outwitHub.extractControls.extractors["scraped"].extractionFinished = true;
						}
						if (!scrapedWidget.JSDatasheet.isSorted()) {
							scrapedWidget.JSDatasheet.sortByCol(scrapedWidget.JSDatasheet.getColumns().getColumnByName("ordinal"), ow.data.sheets.DataSheet.ASC_SORT_ORDER);
						}
						outwitHub.UIControls.getWidgetForView("scraped").JSDatasheet.resetDefaultOrder();
						outwitHub.extractControls.extractorManager.stopInstance("scraped", "fastScrape");
            outwitHub.extractControls.checkIfCatch("scraped");
					} catch (e) {
						ow.log.error("Scrape onStop error " + e);
						alert("Scrape onStop error " + e);
					}
					//ow.log.warning("#5 changing widgets.page.browserPilot.fastRunning from "+ widgets.page.browserPilot.fastRunning + " to false")
					if (widgets.page.browserPilot.fastRunning) {
						widgets.page.browserPilot.fastRunning = false;
						widgets.page.browserPilot.lastExplorationEnd = Date.now();
						ow.ui.status.updateInfoBox();
						//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration");
						delete outwitKit.currentExploration;
						widgets.page.resetStacks();
					}
				},
				onProgress: function() {
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					coreKit.SoundService.play(coreKit.SoundService.SND_TICK);
					var len = scrapedWidget.JSDatasheet.getLength();
					if (len <= 10 || len % 50 <= 2) {
						scrapedWidget.XULDatasheet.autoResize();
					}
          outwitHub.extractControls.checkIfCatch("scraped");
				},
				onAbort: function() {
					var scrapedWidget = outwitHub.UIControls.getWidgetForView("scraped");
					scrapedWidget.XULDatasheet.autoResize();
					scrapedWidget.XULDatasheet.noAutoResize = false;
					this._wasAborted = true;
					var messageVariableDictionary = {};
					coreKit.observerService.observe(new coreKit.message("scrape", coreKit.message.TYPE_NOTICE, "abortProcess", 9, messageVariableDictionary));
					// XXX very very very bad
					outwitHub.extractControls.extractors["scraped"].extractionFinished = true;
					delete outwitKit.currentExploration;
					widgets.page.browserPilot.fastRunning = false;
					widgets.page.browserPilot.lastExplorationEnd = Date.now();
					ow.ui.status.updateInfoBox();
					//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration");
					widgets.page.resetStacks();
				},
				onError: function() {

				}
			};

			extractorInstance.addProgressListener(progressListener);

			outwitHub.extractControls.extractors["scraped"].done = true;
			// XXX JC
			var mergedOptions = ow.util.js.copy(widgets.page.extractorManager._registered["scraped"].options);
			if (options) {
				Object.keys(options).forEach(function(optName) {
					mergedOptions[optName] = options[optName];
				});
			}

			if (kOW_DEBUGMODE) {
				ow.log.error("#### About to initiate fast scrape for " + URIList.length + " URIs");
				ow.log.error("#### Current _XHRs.length " + (ow.network.XHRManager._XHRs ? ow.network.XHRManager._XHRs.length : 0));
				ow.log.error("#### Current _groups.length " + (ow.network.XHRManager._groups ? ow.network.XHRManager._groups.length : 0));
			}

			extractorInstance.fromURIs(URIList, forceScraperList, mergedOptions, contextualData);

			if (kOW_DEBUGMODE) {
				ow.log.error("#### executed extractorInstance.fromURIs()");
				ow.log.error("#### Current _XHRs.length " + (ow.network.XHRManager._XHRs ? ow.network.XHRManager._XHRs.length : 0));
				ow.log.error("#### Current _groups.length " + (ow.network.XHRManager._groups ? ow.network.XHRManager._groups.length : 0));
			}

			if (kOW_DEBUGMODE) {
				//locale the message
				var pString = coreKit.localeService.getString("applyingScrapersStop", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
				//replace the name of the automator in the message
				pString = pString.replace("#numberOfLines#", URIList.length);
				ow.log.debug(pString);
			}
		};

		var forceScraperList;
		if (typeof(aScraper) == "number") {
			var scraper = mainKit.automatorManagerService.getTypedManager("scraper").get(aScraper);
			forceScraperList = [scraper];
		} else if (ow.automation.Automator.isPrototypeOf(aScraper)) {
			forceScraperList = [aScraper];
		} else if (Array.isArray(aScraper)) {
			forceScraperList = aScraper;
		} else {
			forceScraperList = false;
		}

		if (forceScraperList) {
			var UIHelper = ow.ui.automation.ManagerHelper.create(mainKit.automatorManagerService.getTypedManager("scraper"));
			UIHelper.decryptAutomator(forceScraperList, false, doApply);
		} else {
			doApply(forceScraperList);
		}
	},


	// ==================================================================================================================
	// ================================================== FAST CONTACTS =================================================
	// ==================================================================================================================


	fastContacts: function(aURIList, forceOptions) {
		//ow.log.warning("#1 changing widgets.page.browserPilot.fastRunning from "+widgets.page.browserPilot.fastRunning+" to true")
		widgets.page.browserPilot.fastRunning = true;
		widgets.page.browserPilot.lastExplorationStart = Date.now();
		ow.ui.status.updateInfoBox();
		// alert("starting with: "+aURIList.length)
		// alert("first element: "+JSON.stringify(aURIList[0]));
		// from beans to nsIURIs
		var URIList = aURIList.map(function(elt) {
			try {
				return elt.URI;
			} catch (e) {
				return null;
			}
		});
		//  alert("filtered: "+URIList.length)
		//  alert("first element: "+JSON.stringify(URIList[0]));
		if (kOW_DEBUGMODE) { //locale the message
			var pString = coreKit.localeService.getString("fastContactsStart", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
			//replace the name of the automator in the message
			pString = pString.replace("#numberOfLines#", URIList.length);
			ow.log.debug(pString);
		}
		//alert(URIList.length+"\n\n"+JSON.stringify(URIList));

		if (!forceOptions || !forceOptions.doNotShowView) outwitHub.UIControls.showView("emails");
		//alert(JSON.stringify(forceOptions));

		var contactsWidget = outwitHub.UIControls.getWidgetForView("emails");
		contactsWidget.pageChange();

		var extractorInstance = outwitHub.extractControls.extractorManager.getInstanceForLocation("emails", "fastContacts");

		var progressListener = {
			_wasAborted: false,
			onStart: function() {
				var contactsWidget = outwitHub.UIControls.getWidgetForView("emails");
				contactsWidget.XULDatasheet.noAutoResize = true;
				this._wasAborted = false;
				var messageVariableDictionary = {};
				coreKit.observerService.observe(new coreKit.message("fastContacts", coreKit.message.TYPE_NOTICE, "startProcess", 6, messageVariableDictionary));
				outwitHub.extractControls.extractors["emails"].extractionFinished = false;
			},
			onStop: function(error) {
				var contactsWidget = outwitHub.UIControls.getWidgetForView("emails");
				contactsWidget.XULDatasheet.autoResize();
				contactsWidget.XULDatasheet.noAutoResize = false;

				try {
					if (!this._wasAborted && !outwitHub.extractControls.extractors["emails"].extractionFinished) {
						var contactsWidget = outwitHub.UIControls.getWidgetForView("emails");
						var eventName = "";
						var messageVariableDictionary = {};
						var maxCount = outwitHub.extractControls.extractors["emails"].maxCountLightVersion;

						messageVariableDictionary['#displayedColumns#'] = contactsWidget.XULDatasheet.innerTree.columns.count;
						messageVariableDictionary['#allColumns#'] = contactsWidget.JSDatasheet.getColumns().getLength();
						if (!ow.app.module.uck.ok() && maxCount != undefined && contactsWidget.XULDatasheet.view.rowCount >= maxCount) {
							messageVariableDictionary['#maxCount#'] = maxCount;
							messageVariableDictionary['#count#'] = contactsWidget.countFound;
							coreKit.observerService.observe(new coreKit.message("emails", coreKit.message.TYPE_NOTICE, "endProcess.fastContacts.limited", 6, messageVariableDictionary));
							coreKit.observerService.observe(new coreKit.message("fastContacts", coreKit.message.TYPE_NOTICE, "endProcess.limited", 9, messageVariableDictionary));
						} else {
							messageVariableDictionary['#count#'] = outwitHub.UIControls.getWidgetForView("emails").XULDatasheet.view.rowCount;
							coreKit.observerService.observe(new coreKit.message("emails", coreKit.message.TYPE_NOTICE, "endProcess.fastContacts", 6, messageVariableDictionary));
							coreKit.observerService.observe(new coreKit.message("fastContacts", coreKit.message.TYPE_NOTICE, "endProcess", 9, messageVariableDictionary));
						}
						outwitHub.extractControls.extractors["emails"].extractionFinished = true;
					}
					// XXX removed, probably useless
					// outwitHub.UIControls.getWidgetForView("emails").JSDatasheet.resetDefaultOrder();

					outwitHub.extractControls.extractorManager.stopInstance("emails", "fastContacts");
          outwitHub.extractControls.checkIfCatch("emails");

					// XXX this should be done automatically, don't know why it's working on scraped and not here
					contactsWidget.endAppend();
				} catch (e) {
					ow.log.error("fastContact onStop error", e);
				}
				//ow.log.warning("#2 changing widgets.page.browserPilot.fastRunning from " + widgets.page.browserPilot.fastRunning + "to false")
				if (widgets.page.browserPilot.fastRunning) {
					widgets.page.browserPilot.fastRunning = false;
					widgets.page.browserPilot.lastExplorationEnd = Date.now();
					ow.ui.status.updateInfoBox();
					var lastTask = widgets.page.currentExplorationTasks.pop();
					if (widgets.page.currentExplorationTasks.length == 0 && !/fastContactsInDomains/.test(lastTask)) {
						//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration")
						widgets.page.resetStacks();
					}
				}
				//alert("stopped-"+widgets.page.currentExplorationTasks);// COMMENT
			},
			onProgress: function() {
        coreKit.SoundService.play(coreKit.SoundService.SND_TICK);
        var datasheet, datasheetLength;
        datasheet = contactsWidget.JSDatasheet;
        datasheetLength = datasheet.getLength();
				if (datasheetLength <= 10 || datasheetLength % 50 <= 2) {
					contactsWidget.XULDatasheet.autoResize();
				}
        outwitHub.extractControls.checkIfCatch("emails");
			},

			onAbort: function() {
				contactsWidget.XULDatasheet.autoResize();
				contactsWidget.XULDatasheet.noAutoResize = false;
				this._wasAborted = true;
				var messageVariableDictionary = {};
				coreKit.observerService.observe(new coreKit.message("fastContacts", coreKit.message.TYPE_NOTICE, "abortProcess", 9, messageVariableDictionary));
				// XXX very very very bad
				outwitHub.extractControls.extractors["emails"].extractionFinished = true;
				widgets.page.currentExplorationTasks.pop();
				if (widgets.page.currentExplorationTasks.length == 0) {
					//widgets.page.emptyStacks("visited, queried, toVisit, currentExploration") ;
					widgets.page.resetStacks();
				}
				//alert("aborted-"+widgets.page.currentExplorationTasks);// COMMENT
			},

			onError: function() {
				//widgets.page.currentExplorationTasks.pop();
				//alert("error-"+widgets.page.currentExplorationTasks);// COMMENT
			}
		};

		extractorInstance.addProgressListener(progressListener);

		outwitHub.extractControls.extractors["emails"].done = true;

		var options = ow.util.js.copy(widgets.page.extractorManager._registered["emails"].options);
		//widgets.page.extractorManager._registered["emails"].options;
		if (forceOptions) {
			Object.keys(forceOptions).forEach(function(option) {
				options[option] = forceOptions[option];
			});
		}
		extractorInstance.fromURIs(URIList, options);

		if (kOW_DEBUGMODE) {
			//locale the message
			var pString = coreKit.localeService.getString("fastContactsStop", "chrome://outwit-kernel/locale/widgets/logger/userLogMessages.properties");
			//replace the name of the automator in the message
			pString = pString.replace("#numberOfLines#", URIList.length);
			ow.log.debug(pString);
		}
	},


	applyMacro: function(aURIList, aMacroId) {
		//alert("outwitHub.extractControls.applyMacro");
		var macroId = (typeof(aMacroId) == "number" ? aMacroId : ow.automation.util.getFirstGenericAutomatorByType("macro").id);
		outwitKit.macroExecutionManager.runMacroFromId(macroId);
	},


};
