<?php
###############################################################################
# sed.php
#
# @author Anil Kumar <akumar@codepunch.com>
# @link   https://codepunch.com
#
############################################################################### 

require 'lib/php/loader.php';

###############################################################################

use CodePunch\Base\Util as UTIL;
use	CodePunch\Base\Text as TEXT;

###############################################################################

if(!UTIL::is_cli()) {
	header("Location: " . UTIL::get_root_url());
	exit;
}

###############################################################################

UTIL::parse_request_data_in_cli();
$rd = UTIL::get_unsafe_request_data_array();

###############################################################################

if(!count($rd)) {
	$version = \CodePunch\Config\Defaults::version_and_build();
	$product = \CodePunch\Config\Defaults::product();
	echo("\n$product $version\n");
	echo("Command Line Tool\n\n");
	echo("Options\n\n");
	echo("-v\nShow version.\n\n");
	echo("dropall\nDrop all tables.\n\n");
	echo("resetdb\nReset all tables except the settings table.\n\n");
	echo("drop=<tablename>\nDrop the specified table.\n\n");
	echo("wipe=<tablename>\nEmpty the specified table.\n\n");
	echo("reset=<tablename>\nReset the specified empty table to default (if any).\n\n");
	echo("index=<indexname> <new|drop|cleanup> table=<tablename> type=<primary|unique|simple> columns=<columnnames>\nCreate a new index, drop an existing index from table or cleanup all manually created indexes.\nThe 'type' and 'columns' are required if creating a new index. The column names should be comma separated.\nIndex name and extra options are not required if 'cleanup' is specified.\nUse cleanup=all to drop indexes on single custom domain columns too.\nYou can also use -s option along with the 'cleanup' to only simulate and not actually drop or create indexes.\n\n");
	echo("list=[tables,users,audit,indexes]\nList all tables, indexes, users or last audit entries.\nOptionally specify table=<tablename> when using this option to get list of indexes to get data for a specific table.\n\n");
	echo("report=<reportname> email=[no,auto]\nEmail a report to the configured address. When set to auto the emailing will be done only if the current time matches with the report scheduler settings.\n\n");
	echo("mailtest=<email> debug=[0,1,2,3,4]\nTest Emailing (please make sure that all Email Settings are completed)\n\n");
	echo("setssl=[clean]\nGrab all subdomain SSL certificates and add them to the global SSL certificate table. If 'clean' is specified the table will be cleaned first by removing all the previously imported, but not manually added, data.\n\n");
	echo("import=<profile> (please make sure that the data provider details are already setup).\n\n");
	echo("add in=<autoquery> to=<category> (add domains in autoquery to a category).\n\n");
	echo("remove in=<autoquery> from=<category> (remove domains in autoquery from a category).\n\n");
	echo("txtscan=<selectorfile> days=[0,1,2..30] hours=[0,1,2..24]\nBrute force scan for TXT records in all domains. Use days or hours to specify that the domains added in the last n days/hours alone are scanned. Use days=0 to scan all domains. Default is 7 days and 0 hours.\n\n");
	echo("sdscan=<selectorfile> days=[0,1,2..30] hours=[0,1,2..24]\nBrute force scan for A/CNAME records in all domains. Use days or hours to specify that the domains added in the last n days/hours alone are scanned. Use days=0 to scan all domains. Default is 7 days and 0 hours.\n\n");
	echo("websites=</path/to/scriptfile> days=[1,2..30] count=[0,1,2..30]\nTake home page webshots of domains that have valid HTTP destination URL.\nOptionally use days=N to specify if a new screenshot should be taken if the earlier version is older than N days (default is 7 days).\nThe 'count' specifies the number of domains that will be processed on each pass (default is 12, setting it to 0 will process all domains).\nThe scriptfile will be executed once for each domain with the domain name as a parameter.\n\n");
	echo("update [--force] [-y]\nCheck for minor version updates. Download the latest version, if available, and unzip into the installation folder after confirming.\nUse --force to download even if the installation is current.\nUse -y to overwrite current installation folder without confirming.\n\n");
	exit;
}

###############################################################################

if(UTIL::is_request_key_set("dropall")) {
	UTIL::confirm_action("About to drop all tables!!!");
	$auth = new \CodePunch\Config\Auth(\CodePunch\DB\Database::DROP_TABLES);
	exit;
}

###############################################################################

try {
	$auth = new \CodePunch\Config\Auth(\CodePunch\DB\Database::READONLY_TABLES);
	$db = $auth->getDatabase();
}
catch(Exception $e) {
	$logger = new \CodePunch\Base\CPLogger();
	$logger->error($e->getMessage());
	UTIL::print($e->getMessage());
	exit;
}

foreach($rd as $key=>$value) {
	$key = strtolower($key);
}

if(!$db) {
	UTIL::print("Unable to connect to database. Please open in browser to check if license has expired.");
	exit;
}

###############################################################################

try {
	if(isset($rd['drop']) && !isset($rd['index'])) {
		$table = $db->getRealTableName($rd['drop']);
		$schemaManager = $db->getConnection()->getSchemaManager();
		if ($schemaManager->tablesExist(array($table)) !== true) {
			UTIL::print("Table $table not found");
		}
		else {
			UTIL::confirm_action("About to drop $table");
			if($db->dropTable($table)) {
				UTIL::print("Table $table dropped");
				exit;
			}
		}
	}

	###############################################################################

	if(isset($rd['wipe'])) {
		$table = $db->getRealTableName($rd['wipe']);
		$schemaManager = $db->getConnection()->getSchemaManager();
		if ($schemaManager->tablesExist(array($table)) !== true) {
			UTIL::print("Table $table not found");
		}
		else {
			UTIL::confirm_action("About to wipe $table");
			if($db->truncateTable($table)) {
				UTIL::print("Table $table wiped.");
				exit;
			}
		}
	}

	###############################################################################

	if(isset($rd['list'])) {
		$what = $rd['list'];
		if($what == "tables") {
			$tables = $db->getAllTableNames();
			if($tables !== false && is_array($tables)) {
				foreach($tables as $table) {
					$rows = $db->getFromTable("count(*) as count", $table);
					if($rows !== false && isset($rows[0])) {
						$rows[0] = array_change_key_case($rows[0], CASE_LOWER);
						$count = $rows[0]['count'];
						UTIL::print("$table: $count rows");
					}
					else
						UTIL::print($table);
				}
			}
		}
		else if($what == "users") {
			$users = $db->getFromTable("name", $db->getUserTableName());
			if($users !== false && is_array($users)) {
				foreach($users as $user) {
					$user = array_change_key_case($user, CASE_LOWER);
					UTIL::print($user['name']);
				}
			}
		}
		else if($what == "audit") {
			$keyword = isset($rd['q']) ? $rd['q'] : "";
			$data = $db->getAuditData($auth, 50, $keyword);
			$index = 1;
			foreach($data as $d) {
				UTIL::print("$index] $d");
				$index++;
			}
		}
		else if($what == "indexes" || $what == "index") {
			$tname = "";
			if(isset($rd['table']))
				$tname = $db->getRealTableName($rd['table']);
			$type = "";
			if(isset($rd['type'])) {
				$type = $rd['type'];
				if(UTIL::in_array_casei($type, array('simple','index','unique','primary'))) {
					if($type == "index")
						$type = "simple";
				}
				else
					$type = "";
			}
			$data = $db->getIndexData($tname);
			$index = 1;
			$alltables = array();
			foreach($data as $table=>$d) {
				$alltables[] = $table;
				$idxc = 1;
				foreach($d as $di) {
					if($type != "") {
						if(!$di[$type])
							continue;
						if(!strcasecmp($type, "unique") && $di['primary'])
							continue;
					}
					
					if($idxc == 1) {
						UTIL::print("$index] $table\n");
						$alltables = array_diff($alltables, array($table));
						$index++;
					}
					
					$prop = array();
					if($di['simple']) $prop[] = "simple";
					if($di['unique']) $prop[] = "unique";
					if($di['primary']) $prop[] = "primary";
					UTIL::print("\t$idxc\tName: " . $di['name']);
					UTIL::print("\t\tType: " . implode(" ", $prop));
					UTIL::print("\t\tColumns: " . implode(", ", $di['columns']) . "\n");
					$idxc++;
				}
			}
			foreach($alltables as $etable) {
				UTIL::print("$index] $etable\n");
				$index++;
			}
		}
	}

	###############################################################################

	if(isset($rd['index'])) {
		$indexname = $rd['index'];
		if(isset($rd['table']) && isset($rd['drop'])) {
			$realtablename = $db->getRealTableName($rd['table']);
			if($realtablename != "") {
				UTIL::confirm_action("Dropping the index $indexname in $realtablename. This can't be undone.");
				if($db->deleteIndex($realtablename, $indexname)) {
					UTIL::print("$indexname dropped from $realtablename.");
				}
				else
					UTIL::print(\CodePunch\Base\RecentlyLoggedErrors::getLast());
			}
			else
				UTIL::print("Please specify a valid table name.");
		}
		else if(isset($rd['table']) && isset($rd['new']) && isset($rd['columns']) && isset($rd['type'])) {
			$realtablename = $db->getRealTableName($rd['table']);
			$columns = implode("|", explode(",", $rd['columns']));
			$type = $rd['type'];
			if(UTIL::in_array_casei($type, array('simple','index','unique','primary'))) {
				if($type == "simple")
					$type = "index";
				if($type == "unique")
					UTIL::confirm_action("Creating a unique index!\nThis could add extra constraints that prevent proper functioning of the database table and the application.");
				if($db->createSQLIndex($realtablename, $type, $columns, $indexname) === true) {
					UTIL::print("$indexname created for $realtablename.");
				}
				else
					UTIL::print(\CodePunch\Base\RecentlyLoggedErrors::getLast());
			}
			else
				UTIL::print("Invalid index type.");
		}
		else if(isset($rd['cleanup'])) {
			$realtablename = "";
			$ignorecustomdomaincolumns = true;
			if(!strcasecmp($rd['cleanup'], "all"))
				$ignorecustomdomaincolumns = false;
			$simulate = false;
			if(isset($rd['-s']))
				$simulate = true;
			if(isset($rd['table']))
				$realtablename = $db->getRealTableName($rd['table']);
			$tablemsg = $realtablename == "" ? "" : "from table $realtablename.";
			UTIL::confirm_action("\nIndex Cleanup $tablemsg\nRemoving non-default indexes and restoring the defaults.\nThis can't be undone.");
			$db->cleanupSQLIndexes($realtablename, $ignorecustomdomaincolumns, $simulate);
		}
		else
			UTIL::print("Invalid parameters.");
	}

	###############################################################################

	if(isset($rd['reset'])) {
		$table = $db->getTableNameWithoutPrefix($rd['reset']);
		$realtablename = $db->getRealTableName($table);
		$tabledatafolder = UTIL::get_install_folder_path() . "lib/php/CodePunch/DB/Init/";
		UTIL::confirm_action("About to reset $realtablename to defaults.");
		if($db->truncateTable($realtablename))
			$db->updateInitDataFromFile($tabledatafolder, array($table));
		$db->runStartupQueries();
	}

	###############################################################################

	if(isset($rd['resetdb'])) {
		$tabledatafolder = UTIL::get_install_folder_path() . "lib/php/CodePunch/DB/Init/";
		$tables = $db->getAllTableNames();
		if($tables !== false && is_array($tables)) {
			UTIL::confirm_action("About to reset all data tables");
			foreach($tables as $realtablename) {
				$tablename = $db->getTableNameWithoutPrefix($realtablename);
				// Don't truncate categories because it will affect 'All Domains' id.
				if(strtolower($tablename) != "settings" && strtolower($tablename) != "categories") {
					UTIL::print("Truncating table $realtablename");
					if($db->truncateTable($realtablename))
						$db->updateInitDataFromFile($tabledatafolder, array($tablename));
				}
			}
			$db->runStartupQueries();
			exit;
		}
	}
	
	###############################################################################

	if(isset($rd['import'])) {
		$apiprofile = $rd['import'];
		$importManager = new \CodePunch\DB\ImportManager($auth);
		$importManager->importDomainsFromRegistrar($apiprofile);
	}
	
	###############################################################################

	if(isset($rd['setssl'])) {
		$db = $auth->getDatabase();
		$clean = (strtolower($rd['setssl']) == "clean");
		$added = $db->importSSLCertsFromSubdomains($auth, $clean);
		echo "$added SSL certificates found and added.\n";
	}
	
	###############################################################################
	
	if(isset($rd['add'])) {
		$category = UTIL::get_sanitized_request_string("to", "");
		$autoquery = UTIL::get_sanitized_request_string("in", "");
		addOrRemoveDomains($auth, $autoquery, $category, true);
	}
	
	###############################################################################
	
	if(isset($rd['remove'])) {
		$category = UTIL::get_sanitized_request_string("from", "");
		$autoquery = UTIL::get_sanitized_request_string("in", "");
		addOrRemoveDomains($auth, $autoquery, $category, false);
	}
	
	###############################################################################

	if(isset($rd['txtscan'])) {
		$days = UTIL::get_request_data_integer("days", 7, 0, 30);
		$hours = UTIL::get_request_data_integer("hours", 0, 0, 24);
		$append_dk = UTIL::is_request_key_set("nodk") ? false : true;
		$luManager = new \CodePunch\LU\LookupManager($auth);
		$DNS = new \CodePunch\LU\DNS($luManager);
		$selectors = $rd['txtscan'];
		$domain = UTIL::get_sanitized_request_string("domain", "");
		$DNS->scanForTXTRecords($selectors, $append_dk, $days, $hours, $domain);
	}
	
	###############################################################################

	if(isset($rd['sdscan'])) {
		$days = UTIL::get_request_data_integer("days", 7, 0, 30);
		$hours = UTIL::get_request_data_integer("hours", 0, 0, 24);
		$luManager = new \CodePunch\LU\LookupManager($auth);
		$DNS = new \CodePunch\LU\DNS($luManager);
		$selectors = $rd['sdscan'];
		$domain = UTIL::get_sanitized_request_string("domain", "");
		$DNS->scanForSubdomains($selectors, $days, $hours, $domain);
	}
	
	###############################################################################

	if(isset($rd['webshot']) || isset($rd['websites'])) {
		$fp = UTIL::get_lock_file("websites");
		if($fp) {
			$scriptfile = isset($rd['webshot']) ? $rd['webshot'] : $rd['websites'];
			if($scriptfile == "") {
				$datafolder = "websites";
				$nodejsfile = "websites.js";
				$outfolder = UTIL::get_log_folder_path() . $datafolder . DIRECTORY_SEPARATOR;
				if(is_dir($outfolder)) {
					if(!is_file($outfolder . $nodejsfile)) {
						copy(UTIL::get_install_folder_path() . 'lib' . DIRECTORY_SEPARATOR . 'js' . DIRECTORY_SEPARATOR . $nodejsfile, $outfolder . $nodejsfile);
					}
				}
				if(!is_file($outfolder . $nodejsfile)) {
					// Support version 4 webshots settings and folders
					$datafolder = "webshots";
					$outfolder = UTIL::get_log_folder_path() . $datafolder . DIRECTORY_SEPARATOR;
					$nodejsfile = "shot.js";
					if(!is_file($outfolder . $nodejsfile))
						UTIL::print("Missing script file for webshot generation.");
				}
				if(is_file($outfolder . $nodejsfile)) {
					chdir($outfolder);
					$scriptfile = "node --unhandled-rejections=strict $nodejsfile -u http://{{DOMAIN}}";
					if(isset($rd['nojs']))
						$scriptfile .= " --nojs";
					if(isset($rd['allcookies']))
						$scriptfile .= " --ac";
					//if(isset($rd['wait']))
					//	$scriptfile .= " --wunwi";
				}
			}
			else {
				if(!is_file($scriptfile)) {
					UTIL::print("Missing script file for webshot generation.");
					$scriptfile = "";
				}
			}
			if($scriptfile != "") {
				$db = $auth->getDatabase();
				$domain = UTIL::get_sanitized_request_string("domain", "");
				$days = UTIL::get_request_data_integer("days", 7, 0, 30);
				$maxcount = UTIL::get_request_data_integer("count", 12, 0, 30);
				if($maxcount == 0 && $days == 0) $maxcount = 50;
				$dc = date("Y-m-d H:i:s", time()-($days*24*3600));
				$sql = "(websites_checked_at IS NULL OR websites_checked_at < ?) AND home_page_url IS NOT NULL";
				$params = array($dc);
				if($domain != "") {
					if(UTIL::starts_with($domain, "[") && UTIL::ends_with($domain, "]")) {
						$sql .= " AND (" . $db->customExpandWhere($domain) . ")";
					}
					else if(strtolower($domain) != "all") {
						$params[] = $domain;
						$sql .= " AND domain=?";
					}
				}
				else {
					$sql .= " AND " . $db->customExpandWhere("([IN_Business Domains] OR [IN_Important Domains])");
				}
				$rows = $db->getFromTable("sid,domain,websites_checked_at,redirect_last_url", $db->getDomainTableName() . " d", $sql, $params, 'websites_checked_at', 'asc');
				if($rows !== false) {
					$count = count($rows);
					if(!$count)
						UTIL::print("No domains found");
					$index = 0;
					foreach($rows as $di) {
						$di = array_change_key_case($di, CASE_LOWER);
						$domain = $di['domain'];
						$url = $di['redirect_last_url'];
						$sid = $di['sid'];
						$index++;
						$perc = intval(($index/$count)*100);
						UTIL::print("\t($perc%)\t$index/$count] $domain");
						takeWebshot($auth, $datafolder, $scriptfile, $sid, $domain, $url);
						if($index >= $maxcount && $maxcount)
							break;
					}
				}
				else
					UTIL::print(\CodePunch\Base\RecentlyLoggedErrors::getLast());
			}
			flock($fp, LOCK_UN);
			fclose($fp);
		}
		else {
			UTIL::print("Unable to get lock file");
		}
	}

	###############################################################################

	if(isset($rd['mailtest'])) {
		$toemail = $rd['mailtest'];
		if(filter_var($toemail, FILTER_VALIDATE_EMAIL)) {
			$debuglevel = isset($rd['debug']) ? $rd['debug'] : 0;
			if(intval($debuglevel) < 0 || intval($debuglevel) > 4)
				$debuglevel = 0;
			$recipients = array($toemail=>'');
			$mailer = new \CodePunch\Base\Mailer($auth);
			$status = $mailer->send(array($recipients), "Test Email", "This is a test email from Watch My Domains SED", "", $debuglevel);
			if($status !== false) {
			}
		}
	}

	###############################################################################

	if(isset($rd['report'])) {
		$responce = array();
		$reportname = $rd['report'];
		$reportManager = new \CodePunch\UI\ReportManager($auth);
		$db = $auth->getDatabase();
		$table = $db->getReportSchedulerTableName();
		$doemail = \CodePunch\UI\ReportManager::SEND_EMAIL_YES;
		if(isset($rd['email'])) {
			$emode = strtolower($rd['email']); 
			if($emode == "auto")
				$doemail = \CodePunch\UI\ReportManager::SEND_EMAIL_AUTO;
			else if($emode == "no")
				$doemail = \CodePunch\UI\ReportManager::SEND_EMAIL_NO;
		}
		if($db->hasRow($table, "name", $reportname)) {
			$reportManager->getReport($reportname, $doemail, false, $responce);
			if(isset($responce['report']))
				unset($responce['report']);
			$emailed = UTIL::get_from_array($responce['emailed'], 0);
			UTIL::print($responce);
			if($emailed)
				UTIL::print("The report was emailed.");
		}
		else if($reportname == "SSL Browser Report") {
			$reportManager->getSSLCertReport($responce, false, $doemail);
			$reportManager->createSSLBrowserReport();
		}
		else
			UTIL::print("'$reportname' not found.");
	}

	###############################################################################

	if(isset($rd['-v'])) {
		$version  = \CodePunch\Config\Defaults::version_and_build();
		UTIL::print(\CodePunch\Config\Defaults::product());
		UTIL::print("Version $version");
	}

	###############################################################################

	if(isset($rd['update'])) {
		$thisbuild = \CodePunch\Config\Defaults::BUILD_DATE;
		if(method_exists($auth, "getDownloadURL")) {
			$verurl = $auth->getDownloadURL("ver");
			$dnloadurl = $auth->getDownloadURL("zip");
		}
		else {
			$dnloadurl = "https://domainpunch.com/sed/wmdsed60_trial.zip.php";
			$verurl = "$dnloadurl?version";
		}
		$verdata = file_get_contents($verurl);
		$verdata = json_decode($verdata, true);
		if(isset($verdata['date'])) {
			$thatbuild = $verdata['date'];
			if(strlen($thatbuild) == 6) {
				$latest = mktime(0,0,0,
					intval(substr($thatbuild, 0, 2)),
					intval(substr($thatbuild, 2, 2)),
					intval(substr($thatbuild, 4, 2)) + 2000);
				$current = strtotime($thisbuild);
				UTIL::print("Latest Version Build Date: " . date("Y-M-d", $latest));
				UTIL::print("Installed Version Build Date: " . date("Y-M-d", $current));

				$m1 = $verdata['majorversion'];
				$m2 = $verdata['minorversion'];
				$newver = $m1/100 . "." . $m2/100;
				$availableversion = $newver . "." . $verdata['buildindex'] . ", " . date("M d Y", $latest);
				$setup = new \CodePunch\Config\Settings($auth);
				$setup->setOption("available_app_build_date", $availableversion);

				if($latest > $current || isset($rd['--force'])) {
					UTIL::print("Downloading the latest version.");
					$fp = UTIL::get_lock_file("update");
					if($fp) {
						if(isset($rd['folder']) && is_dir($rd['folder'])) {
							$basefolder = realpath($rd['folder']) . DIRECTORY_SEPARATOR;
						}
						else {
							$installfolder = UTIL::get_install_folder_path();
							$basefolder = realpath($installfolder . "../") . DIRECTORY_SEPARATOR;
						}
						if(is_dir($basefolder)) {
							$zipfile = $basefolder. "wmdsed60_latest.zip";
							$util = new \CodePunch\base\Util();
							$retv = $util->curl_download_file($dnloadurl, $zipfile);
							if($retv['status'] == 200) {
								UTIL::print("Latest version downloaded to $zipfile");
								if(UTIL::ends_with(strtolower($basefolder), "dev" . DIRECTORY_SEPARATOR))
									UTIL::print("This appears to be a development folder, exiting.");
								else {
									if(!isset($rd['-y']))
										UTIL::confirm_action("About to overwrite the installation folder \"$basefolder\" with new version!");
									if(UTIL::extractZip($zipfile, $installfolder, "wmdsed60", array(new CodePunch\Base\Util(), 'zip_extract_callback'))) {
										UTIL::print("Files extracted into $installfolder");
										unlink($zipfile);
									}
									else
										UTIL::print("Unable to extract ZIP file. Check if the zip extension is installed.");
								}
							}
							else if(isset($retv['error']))
								UTIL::print($retv['error']);
						}
						else
							UTIL::print("Unable to find a folder to download to");

						flock($fp, LOCK_UN);
						fclose($fp);
					}
					else {
						UTIL::print("Unable to get lock file");
					}
				}
			}
		}
		exit;
	}
}
catch(Exception $e) {
	$logger = new \CodePunch\Base\CPLogger();
	$logger->error($e->getMessage());
	UTIL::print($e->getMessage());
	exit;
}

###############################################################################
	
function takeWebshot($auth, $foldername, $scriptfile, $sid, $domain, $url) {
	$db = $auth->getDatabase();
	$cookiefolder = UTIL::get_log_folder_path() . $foldername . DIRECTORY_SEPARATOR . "cookies" . DIRECTORY_SEPARATOR;
	$start_ts = date('Y-m-d H:i:s');
	$cm = $auth->getProClass('Cookies');
	// Find all subdomains
	$badsubdomains = ['mail'];
	for($i = 0; $i < 20; $i++) {
		$badsubdomains[] = 'ns' . (string)$i;
		$badsubdomains[] = 'dns' . (string)$i;
	}
	$query = "d.sid = ? AND s.auto_added = 0 AND s.subdomain NOT IN (?)";
	$select = "s.hid,s.subdomain,s.hostname";
	$iglist = array($badsubdomains, \Doctrine\DBAL\Connection::PARAM_STR_ARRAY);
	$data = $db->getDomainReportRows($query, array($sid, $iglist), $select, "d.sid", "asc");
	if($data !== false && is_array($data) && count($data)) {
		foreach($data as $di) {
			$current_ts = date("Y-m-d H:i:s");
			$hostname = str_replace("@.", "", $di['s_hostname']);
			$subdomain = $di['s_subdomain'];
			$sd = str_replace("@", "", $subdomain);
			$ascii_hostname = UTIL::idn_convert_to_host_name($domain, $sd);
			UTIL::print("\t\t".$hostname . ": " . $ascii_hostname);
			$command = $scriptfile;
			if(stripos($command, "{{DOMAIN}}") !== false)
				$command = str_ireplace("{{DOMAIN}}", $ascii_hostname, $command);
			else
				$command .= " $ascii_hostname";

			exec($command, $output, $retval);
			$db->deleteFromTable($db->getSubdomainTableName(), "(auto_added=? OR auto_added = ?) AND sid=? AND subdomain=?", array(\CodePunch\LU\LookupManager::SD_WEBSHOT_ROWS, \CodePunch\LU\LookupManager::SD_COOKIE_ROWS, $sid, $subdomain));
			$da = array('sid'=>$sid, 'auto_added'=>\CodePunch\LU\LookupManager::SD_WEBSHOT_ROWS, 'added_on'=>$current_ts, 'subdomain'=>$subdomain);
			$db->insertIntoTable($db->getSubdomainTableName(), $da);
			
			if($cm) {
				$jsonfile = $cookiefolder . $ascii_hostname . ".json";
				$cm->save($sid, $url, $domain, $subdomain, $current_ts, $jsonfile);
			}
		}
	}

	$domain_data = array();
	$domain_data['domain'] = $domain;
	$domain_data['websites_checked_at'] = $start_ts;
	$db->updateDomainTable($domain_data);
}

###############################################################################

function addOrRemoveDomains($auth, $autoquery, $category, $copy=true) {
	$db = $auth->getDatabase();
	
	// Check if category and autoquery exist.
	$cid = $db->getCategoryID($category);
	if($cid <= 1) 
		throw new Exception("Invalid Category Name: " . $category);
	
	$aqwhere = $db->findOneOf($db->getAutoQueryTableName(), "name", $autoquery, "query");
	if($aqwhere === false) 
		throw new Exception("Invalid Autoquery Name: " . $autoquery);
	$aqparams = $db->findOneOf($db->getAutoQueryTableName(), "name", $autoquery, "params");
	$pcount = substr_count($aqwhere, "?");
	$aqparams = str_replace(array("\n","\r","\t"), ",", $aqparams);
	$aqp = array_filter(explode(",", $aqparams));
	if(count($aqp) == $pcount) {
		$aqwhere = "(" . $aqwhere . ")";
		if($copy)
			$aqwhere .= " AND ([NOTIN_" . $category . "])";
		else
			$aqwhere .= " AND ([IN_" . $category . "])";
		$aqwhere = $db->customExpandWhere($aqwhere);
		$grid = new \CodePunch\UI\DataGrid($auth);
		$dpp = 1000;
		$found = 0;
		$page = 1;
		$actioncount = 0;
		do {
			$totalcount = 0;
			$domains = [];
			$ids = [];
			$responce = $grid->jqGridDomainTableData($db->getDomainTableName(), "sid", "sid,domain", "", "asc", 0, $aqwhere, $aqp, array(), $page, $dpp);
			$responce = json_decode(json_encode($responce), true);
			$totalcount = intval($responce['records']);
			if($totalcount) {
				$rows = $responce['rows'];
				foreach($rows as $row) {
					$domain = $row['domain'];
					$domains[] = $domain;
					$ids[] = $row['sid'];
				}
				if($copy)
					$actioncount += $db->addDomainIDsToCategoryByID($ids, $cid);
				else {
					foreach($ids as $sid) {
						$status = $db->removeDomainIDFromCategoryByID($sid, $cid);
						$actioncount += $status ? 1 : 0;
					}
				}
				$found += count($rows);
				$page++;
			}
			else
				break;
		}
		while($found < $totalcount);
		if($copy)
			UTIL::cli_print("$found domains found, $actioncount domains added to $category");
		else
			UTIL::cli_print("$found domains found, $actioncount domains removed from $category");
		$db->setCategoryCounts();
	}
}

###############################################################################

?>
