<?php
abstract class grabber
{
    public $proxy = null;
    //public $limit = 200;
    public $limit = 5000;
    public $fails = 7;//15;
    public $site_id = null;

    protected $site;
    protected $timeout = 60;
    protected $data;
    protected $__proxy = null;
    protected $cookies = null;
    protected $startpage = 1;

    protected $check_olditems = true;
    protected $olditems = []; // TODO: private property; protected methods addOldItem, getOldItem
    protected $datelimit;
    protected $cache = [];

    protected $http_tunnel = false;
    
    public function __log($text)
    {
        echo "[" . date("Y-m-d H:i:s") . "] " . $text,"\n";
        return 0;
    }

    protected function error($err)
    {
        $this->session_end();
        die($this->__log($err));
    }

    public function __construct($site_id)
    {
		// $num =  mysql_result(mysql_query("SELECT COUNT(*) FROM grabber.objects WHERE link_id = 0 AND link_s = 0 AND is_agency = 0 AND checked = 0 AND display = 0 AND category = 'kvartiry';"),0);
		// mysql_query("DELETE FROM grabber.objects WHERE link_id = 0 AND link_s = 0 AND is_agency = 0 AND checked = 0 AND display = 0 AND category = 'kvartiry';");
		// $this->__log("\n\n=========\n INCORRECT ITEMS CLEARED: {$num}\n=========\n\n");   	
		
		$res = mysql_fetch_object(mysql_query("SELECT value FROM oris.oris_conf WHERE name = 'GrabberFailsCount';"));
		$failsValueFromDb = $res->value;
		if($failsValueFromDb)
			$this->fails = $failsValueFromDb;
		    
		
        $this->sites = [];
        $dbc = mysql_query("SELECT * FROM sites ORDER BY id");
        while($rt = mysql_fetch_object($dbc)) $this->sites[$rt->id] = $rt;
        
        $site_id = (int) $site_id;
        $this->site = &$this->sites[$site_id];
        if(!$this->site) $this->error("site (id = {$site_id}) not found");
        if(!$this->site->is_active) $this->error("site «" . $this->site->name . "» (id = " . $this->site->id . ") disabled");
        $this->site_id = $this->site->id;
        
        $this->cities  = conf::$cities;
        $this->timeout = conf::$timeout;
        $this->mirrors = conf::$mirrors;
        
        $this->days = [
            "вчера"   => date("Y-m-d",time() - 86400),
            "сегодня" => date("Y-m-d"),
        ];
        //$this->datelimit = date("Y-m-d", time() - 2 * 86400) . " 00:00:00";
        $this->datelimit = date("Y-m-d", time() - 7 * 86400) . " 00:00:00";

        $this->month = [
            "01" => "января",
            "02" => "февраля",
            "03" => "марта",
            "04" => "апреля",
            "05" => "мая",
            "06" => "июня",
            "07" => "июля",
            "08" => "августа",
            "09" => "сентября",
            "10" => "октября",
            "11" => "ноября",
            "12" => "декабря",
        ];

        $this->year = date("Y");
        
        $this->dS = 0.1;
        
        $this->deals = [
            prodam => 1,
            sdam => 2,
        ];
        
        $this->categories = [
            kvartiry => 1, # Квартиры
            komnaty => 2, # Комнаты
            doma_dachi_kottedzhi => 3, # Дома, дачи, коттеджи
            zemelnye_uchastki => 4, # Земельные участки
            kommercheskaya_nedvizhimost => 5, # Коммерческая недвижимость
            garazhi_i_mashinomesta => 6, # Гаражи и машиноместа
            nedvizhimost_za_rubezhom => 7, # Недвижимость за рубежом
        ];
        
        // $this->__proxy = (object) [ip => "", port => ""];
        // $this->proxy = "10.0.0.1:8800";
        // list($this->__proxy->ip,$this->__proxy->port) = explode(":",$this->proxy);
        $this->cookies = "{$_ENV[ROOT]}/../logs/{$this->site->code}.cookie";
    }

    public function update_proxy($n, $status = "", $error = "")
    {
        if(!$this->__proxy) return false;
        $status = mysql_real_escape_string($status);
        $error  = mysql_real_escape_string($error);
        $sql = "UPDATE site_proxy SET date_check = NOW(), last_status = '{$status}', last_error = '{$error}', fails_counter = " . ($n ? "fails_counter + 1" : "0, success_counter = success_counter + 1, last_success = NOW()") . " WHERE site_id = '{$this->site->id}' AND proxy_id = '{$this->__proxy->id}'";
        mysql_query($sql);
	}

	public function get_proxy($proxy = "", $reset = 0)
    {
        // $this->__proxy = (object) [ip => "", port => ""];
        // $this->proxy = "10.0.0.1:8800";
        // list($this->__proxy->ip,$this->__proxy->port) = explode(":",$this->proxy);
		// return $this->proxy;
        
        if($proxy){
            $this->__proxy = mysql_fetch_object(mysql_query("SELECT * FROM proxy WHERE proxy = '" . mysql_real_escape_string($proxy) . "'")) or $this->error("proxy {$proxy} not found!");
        }
        else {
            if($reset) mysql_query("UPDATE site_proxy SET fails_counter = 0, last_status = '' WHERE site_id = '{$this->site->id}' AND is_active");
            
            $this->__proxy = mysql_fetch_object(mysql_query("SELECT p.* FROM proxy p INNER JOIN site_proxy s ON s.site_id = '{$this->site->id}' AND p.id = s.proxy_id WHERE s.is_active AND (s.last_status != '403' OR s.last_status = '403' AND s.date_check + INTERVAL 15 MINUTE < NOW()) ORDER BY RAND() LIMIT 1"));
            if(!$this->__proxy) return $reset ? $this->error("no active proxy in proxy-list!") : $this->get_proxy("", 1);
        }
        $this->proxy = $this->__proxy->proxy;
        list($this->__proxy->ip,$this->__proxy->port) = explode(":",$this->proxy);
		return $this->proxy;
	}
	public $proxylist;	
	
	public function get_proxylist()
	{
		if(is_null($this->proxylist))
		{
			$que = "SELECT value FROM oris.oris_conf WHERE name = 'ProxyListAvito';";
			$result = mysql_result(mysql_query($que),0);
			if($result)
				$this->proxylist = split("\n", $result);
			else
			{
				$result = file_get_contents('https://my.virty.io/proxy_list/proxies.php?hash=19e43d5e1f5af6a6db7d6d9de7b377f3&type=http&format=format1');
				$this->proxylist = split("\n", $result);
			}
		}
		return $this->proxylist;
	}
    
    public function get_date($date, $time = "")
    {
        if(!$time) $time = "00:00";
        $time .= ":00";
        $date = mb_strtolower(trim($date),"utf-8");

        if($this->days[$date]) $date = $this->days[$date];
        else{
            preg_match("/^(\d*)(\D*)(\d*)$/i",$date,$date);

            if(!$date[3]) $date[3] = $this->year;
            elseif(mb_strlen($date[3],"utf-8") < 4) $date[3] = "20" . $date[3];

            if(!isset($this->monthcache[$date[2]])){
                $this->monthcache[$date[2]] = preg_replace("/\p{P}+$/","",trim($date[2]));

                foreach($this->month as $k => $v){
                    if(preg_match("/^" .  preg_quote($this->monthcache[$date[2]],"/") . "/ui",$v)){
                        $this->monthcache[$date[2]] = $k;
                        break;
                    }
                }
            }

            if(mb_strlen($date[1],"utf-8") < 2) $date[1] = "0" . $date[1];
            
            $date[2] = $this->monthcache[$date[2]];
            if($date[2] > date("m") && $date[3] >= $this->year) $date[3] = $this->year - 1;


            $date = "{$date[3]}-{$date[2]}-{$date[1]}";
        }

        return $date . " " . $time;
    }
    
	private function __get_url($url, $opts, $method) {
        sleep(4);
        $log = "get_url (oris-proxy): {$method} : {$url}";
        $this->__log($log);
        
        $params = [
            METHOD => $method,
            URL => $url,
            OPTS => serialize($opts),
        ];
        $content = http_build_query($params);
        $context = stream_context_create([ http => [
            "protocol_version" => 1.1,
            "method"  => "POST",
            "timeout" => 60,
            "header" => [
                "Content-Type: application/x-www-form-urlencoded",
                "Content-Length: " . strlen($content),
                "Connection: close",
            ],
            "content" => $content,
        ]]);        
        $data = file_get_contents("https://office.oris-info.ru/proxy/index.php", false, $context);
        return $data;
    }
	
	public function get_url($url, $proxy = true, $opts = [], $n = 1)
    {
		if(!$url || !preg_match("/^https?[:]\/\//",$url)) return $this->error("get_url: incorrect url = {$url}");
        if ($this->http_tunnel || !$proxy && $n > 2) {
            $this->http_tunnel = true;
            return $this->__get_url($url, $opts, "GET");
        }
        if($n > $this->fails / 3) {
            if($proxy) $proxy = false;
            else{
                $proxy = true;
                $n = 1;
            }
        }
        $log = "get_url: url = {$url}";
        
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_HEADER, false);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch,CURLOPT_ENCODING , "");
		curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
       curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 2);
     curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);

//curl_setopt ($ch, CURLOPT_POST, 1);


        // curl_setopt($ch, CURLOPT_SSLVERSION, 2);
        if ($this->cookies) {
            curl_setopt($ch, CURLOPT_COOKIEFILE, $this->cookies);
            curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookies);
        }
        
        $headers = [];
        $headers = is_array($opts) ? $opts : [$opts];
        $headers[] = "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3";
        $headers[] = "Cache-Control: max-age=0";
        $headers[] = "Connection: keep-alive";
                if(count(array_filter($headers, function($v) {
			return strpos($v, 'User-Agent') !== false;
			}))==0){
				// $headers[] = "User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0";
				$headers[] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.72 Safari/537.36";
				}
        
        if ($proxy) {
            $this->get_proxy($proxy === true ? "" : $proxy);
            curl_setopt($ch, CURLOPT_PROXY, $this->proxy);
            if ($this->__proxy->pass) curl_setopt($ch, CURLOPT_PROXYUSERPWD, $this->__proxy->login . ":" . preg_replace("/\s/","",$this->__proxy->pass));
            
            $log .= ", proxy = {$this->proxy}";
			
		$this->__log('------------------------------------------------------------------------------------------------------------');
		$this->__log($this->proxy);
		$this->__log($this->__proxy->login);
		$this->__log($this->__proxy->pass);
		$this->__log($this->__proxy->login . ":" . preg_replace("/\s/","",$this->__proxy->pass));
		$this->__log('------------------------------------------------------------------------------------------------------------');
            // $headers[] = "X-Forwarded-For: {$this->__proxy->ip}";
        }
        else {
            $this->__proxy = null;
            $this->proxy = null;
            sleep(3);
        }
        
		$this->__log($log);
        
        curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
		
		curl_setopt($ch, CURLINFO_HEADER_OUT, true);        
		$this->__log('------------------------------------------------------------------------------------------------------------');
		$this->__log(json_encode(curl_getinfo($ch)));
		$this->__log('------------------------------------------------------------------------------------------------------------');
		$this->__log(json_encode(curl_getinfo($ch, CURLINFO_COOKIELIST)));
		$this->__log('------------------------------------------------------------------------------------------------------------');
		$ss = curl_exec($ch);
		$er = curl_error($ch);
        $ci = curl_getinfo($ch);
		$this->__log('------------------------------------------------------------------------------------------------------------');
		$this->__log(json_encode(curl_getinfo($ch)));
		$this->__log('------------------------------------------------------------------------------------------------------------');
        
		curl_close($ch);
        
        $f = $er || $ci["http_code"] != 200;
        
		if ($f) {
			$this->__log("{$log} fails (err = {$er}, http_code = {$ci["http_code"]})");
			return $proxy ? $this->get_url($url, false, $opts) : "";
		}
        
		return $ss;
	}
    
    public function post_url($url, $proxy = true, $opts = [], $form = [])
    {
        //$ss =  $this->__get_url($url, $opts, "POST");
        //return $ss ? $ss : $this->error("post_url: oris-proxy-error");
		
		if(!$url || !preg_match("/^https?[:]\/\//",$url)) return $this->error("post_url: incorrect url = {$url}");
        $query = http_build_query($form);
        $log = "post_url: url = {$url}&{$query}";
        
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_POST, 1);
        curl_setopt($ch, CURLOPT_POSTFIELDS, $query);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_TIMEOUT, $this->timeout);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
        
        if ($this->cookies) {
            curl_setopt($ch, CURLOPT_COOKIEFILE, $this->cookies);
            curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookies);
        }
        
        $headers = [];
        $headers = is_array($opts) ? $opts : [$opts];
        $headers[] = "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3";
        $headers[] = "Cache-Control: max-age=0";
        $headers[] = "Connection: keep-alive";
        $headers[] = "User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0";
        $headers[] = "Content-length: " . strlen($query);
        
        curl_setopt($ch, CURLOPT_PROXY, $this->proxy);
        $log .= ", proxy = {$this->proxy}";
        
		$this->__log($log);
        
        curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
        
        sleep(4);
		$ss = curl_exec($ch);
		$er = curl_error($ch);
        $ci = curl_getinfo($ch);
        
		curl_close($ch);
        
        $f = $er || $ci["http_code"] != 200;
        
		if ($f) {
			$this->__log("{$log} fails (err = {$er}, http_code = {$ci["http_code"]})");
			return "";
		}
        
		return $ss;
	}

    public function session_start() { }
    
    public function session_end()
    {
        @unlink("{$_ENV[ROOT]}/../pids/{$this->site->code}-{$_SERVER[argv][2]}-{$_SERVER[argv][3]}-{$_SERVER[argv][4]}.pid");
    }
    
    abstract public function getPages($url, $n = 0);
    abstract public function getItems($url, $n = 0);
    abstract public function getItem($item, $n = 0);
    abstract public function create_url($urlsfx, $p = 1);
    
    final public function checkItem($item)
    {
        if($item->date && $item->date <= $this->datelimit) return 0;

        $olditem = mysql_fetch_object(mysql_query("SELECT * FROM objects WHERE site_id = '{$this->site->id}' AND id = '{$item->id}'"));
        if(!$this->check_olditems) return $olditem ? 0 : 1;
		
        if( !$olditem ) return 1;
        if( !$olditem->display ||
            $olditem->is_agency == 1 ||
            $item->date <= $olditem->date ||
            $olditem->category != $this->category ||
            $olditem->deal_type != $this->deal_type
			// ||            $olditem->category != "kvartiry" && $olditem->category != "komnaty"
			// && $olditem->category != "doma_dachi_kottedzhi"
			// && $olditem->category != "zemelnye_uchastki"
        ) return 0;
        
        
        $d1 = date_create($item->date);
        $d2 = date_create($olditem->date);

        $r = date_diff($d1, $d2);
		
        if(!$r->days) return 0;
        
        $olditem->date_diff = $r->days;
        $olditem->price = __floatval(preg_replace('/[^\d.,]+/', '', $olditem->price));
        
        $this->olditems[$item->id] = $olditem;
        return 1;
    }
    
    public function saveItem($item)
    {
        if (isset($this->olditems[$item->id])) return $this->updateItem($item);

        $item->site_id = $this->site->id;
        $item->date_add = date("Y-m-d H:i:s");

        if(!$item->city) $item->city = $this->city;
        if(!$item->deal_type) $item->deal_type = $this->deal_type;
        if(!$item->category) $item->category = $this->category;
        
 
$name=[
'АРЕВЕРА-Недвижимость',
'Компания «Этажи»',
'ГРАНТА-недвижимость',
'Lacrum-Недвижимость',
'Агентство недвижимости \"Астория\"',
'Паритет-инвест',
'Исполин Недвижимость',
'Красноярское Инвестиционное Агентство Недвижимости',
'Novostroy Invest эксперт на рынке Недвижимости Красноярска!',
'Century21Smart',
'Агентство недвижимости «Фрегат»',
'ООО \"Алекс\"',
'Центр обмена жилья',
'Агентство недвижимости \"СВОЁ\"',
'AN BROOKLIN',
'ООО ГАРАНТ',
'АНГОР',
'Агентство недвижимости и оценки \"Профессионал\"',
'АН ДОМ',
'Красноярский Центр Недвижимости',
'Центр Загородной недвижимости \"Кедр\"',
'Ремесленникъ',
'Перспектива24-Красноярск-Федеральный оператор недвижимости',
'АН \"СИТИ\"',
'АН \"Stolica-24\"',
'ООО \"ЦЕНТР ОФОРМЛЕНИЯ НЕДВИЖИМОСТИ\"',
'\"Делегат Групп\" недвижимость',
'АН \"Экспонента\"',
'АН Территория 24',
'АКТИВ НЕДВИЖИМОСТЬ',
'АН Городские районы',
'АН ИнвестКристал',
'АН Городские районы',
'ЖилДом - Недвижимость !',
'Новый Красноярск',
'АН СССР',
'Агентство Недвижимости Ангерона',
'Ankom-недвижимость',
'АКТИВ НЕДВИЖИМОСТЬ',
'Сибирский дом',
'Бюро Недвижимости \"МеЧтА\"',
'Авангард+',
'АН \"Вариант Центр\"',
'Риэлт Сибирь',
'агентство недвижимости \"Ермак\"',
'Группа компаний РиМиР',
'Дельта Гарант'];

for ($i=0; $i<=count($name); $i++)
{
if($item->seller === $name[$i])
{
                              $item->display = 0;
//print_r($name);          

               }
}


	if($item->is_agency === -1){
			$phoneNotProtectedAndInOrisAgenciesInLastFourMonths = 
			 !preg_match("/(Номер продавца защищён|Номер агентства защищён|Номер защищён)/ismU",$item->description) &&
			 mysql_fetch_object(mysql_query("SELECT COUNT(*) as cnt FROM oris.oris_agencies WHERE phone = '{$item->phone}' AND date_change > DATE_ADD(NOW(), INTERVAL -8 MONTH);"))->cnt > 0;			 
			 if($phoneNotProtectedAndInOrisAgenciesInLastFourMonths){				 
				$item->display = 0;
			 }
		}
		
        $data = (array) $item;
        unset($data["city"]);
        unset($data["date_begin"]);
        unset($data["date_end"]);
        unset($data["proxy"]);
        unset($data["session_id"]);
        unset($data["raw_params"]);
        foreach ($data as &$_) $_ = mysql_real_escape_string($_);
        unset($_);
		
        # TODO: ON DUPLICATE KEY UPDATE ...		
        $sql = "INSERT INTO objects (`" . implode("`, `", array_keys($data)) . "`) VALUES('" . implode("', '", $data) . "')";
        mysql_query($sql) or $this->error(mysql_error());
		
        
        if($item->display){
            $item->object_id = mysql_insert_id();
            if($item->phone > ""){
                $tmp = explode("\n",preg_replace("/^7/sm","8",$item->phone));
                $sql = "INSERT INTO objects_phones (object_id,phone) VALUES ";
                foreach($tmp as $_){
                    $_ = mysql_real_escape_string(trim($_));
                    $sql .= "('{$item->object_id}','{$_}'), ";
                }
                $sql = preg_replace("/,\s+$/","",$sql);
                $sql .= " ON DUPLICATE KEY UPDATE object_id = object_id";
                mysql_query($sql) or $this->__log("saveItem(): " . mysql_error());
            }
			
            
            $this->get_street($item);
            $this->get_house_number($item);
            $this->link_object($item);
			
            mysql_query("UPDATE objects SET link_id = '{$item->link_id}', link_s = '{$item->link_s}', street_id = '{$item->street_id}', streets = '{$item->streets}', house_number = '{$item->house_number}' WHERE object_id = '{$item->object_id}'");
            
            mysql_query("UPDATE objects_phones SET link_id = '{$item->link_id}' WHERE object_id = '{$item->object_id}'");
        }
    }
    
    public function updateItem($item)
    {
        $olditem = $this->olditems[$item->id];
        
        $item->site_id = $this->site->id;
        $item->s = number_format($item->s, 2, '.', '');
        $date_add = date("Y-m-d H:i:s");

        if(!$item->city) $item->city = $this->city;
        if(!$item->deal_type) $item->deal_type = $this->deal_type;
        if(!$item->category) $item->category = $this->category;
        
        unset($item->city);
        unset($item->date_begin);
        unset($item->date_end);
        unset($item->proxy);
        unset($item->session_id);
        unset($item->raw_params);
				
		$urlContainsCian = strpos($item->url, 'cian.ru') !== false;
		$urlContainsIrr = strpos($item->url, 'irr.ru') !== false;
		
		$excludeUpdate = $urlContainsCian || $urlContainsIrr;
		if($excludeUpdate){
			return;
		}
	
		
		if($item->is_agency === -1){
			$phoneNotProtectedAndInOrisAgenciesInLastFourMonths = 
			 !preg_match("/(Номер продавца защищён|Номер агентства защищён|Номер защищён)/ismU",$item->description) &&
			 mysql_fetch_object(mysql_query("SELECT COUNT(*) as cnt FROM oris.oris_agencies WHERE phone = '{$item->phone}' AND date_change > DATE_ADD(NOW(), INTERVAL -8 MONTH);"))->cnt > 0;
			 if($phoneNotProtectedAndInOrisAgenciesInLastFourMonths){
				$item->display = 0;
			 }
			 else{
				 $note = mysql_fetch_object(mysql_query("SELECT note as note FROM grabber.objects_notes WHERE (link_id, link_s) IN (SELECT link_id,link_s FROM grabber.objects WHERE object_id = '{$olditem->object_id}');"))->note;
				 if($note && preg_match("/([Аа]гентство)/ismU",$note)){
					 $item->display = 0;				 
				 }	
			 }
		}
        
        $cmpfields = ['date' => 0, 'price' => 0, 'phone' => 0, 'level' => 0, 'levels' => 0, 'rooms' => 0, 's' => 0, 'address' => 0];
        $log = "UPDATED: [{$olditem->object_id}] : ";
        $sql = "UPDATE objects SET date_add = '{$date_add}'";
        foreach((array) $item as $k => $v) {
            if(isset($cmpfields[$k])) {
                if($k == 'phone') {
                    $t1 = explode("\n",preg_replace("/^7/sm","8", $item->phone));
                    $t2 = explode("\n",preg_replace("/^7/sm","8", $olditem->phone));
                    $d = array_diff($t1, $t2);
                    $cmpfields[$k] = count($d);
                }
                else if($item->$k != $olditem->$k) {
                    $log .= "{$k}: {$olditem->$k} --> {$item->$k}; ";
                    $cmpfields[$k] = 1;
                }
            }
            else if($item->$k != $olditem->$k) {
                $log .= "{$k}; ";
            }
            $sql .= ", `{$k}` = '" . mysql_real_escape_string($v) . "'";
        }
        $sql .= " WHERE object_id = '{$olditem->object_id}'";
        
        mysql_query($sql) or $this->error(mysql_error());
        $this->__log($log);
		
        if($item->display) {
            $item->date_add = $date_add;
            $item->object_id = $olditem->object_id;
            
            if($item->phone > ""){
                $tmp = explode("\n",preg_replace("/^7/sm","8",$item->phone));
                
                mysql_query("DELETE FROM objects_phones WHERE object_id = '{$item->object_id}'");
                $sql = "INSERT INTO objects_phones (object_id,phone) VALUES ";
                foreach($tmp as $_){
                    $_ = mysql_real_escape_string(trim($_));
                    $sql .= "('{$item->object_id}','{$_}'), ";
                }
                $sql = preg_replace("/,\s+$/","",$sql);
                $sql .= " ON DUPLICATE KEY UPDATE object_id = object_id";
                mysql_query($sql) or $this->__log("updateItem(): " . mysql_error());
            }
            
            $this->get_street($item);
            $this->get_house_number($item);
            $this->link_object($item, $cmpfields);
			
            mysql_query("UPDATE objects SET link_id = '{$item->link_id}', link_s = '{$item->link_s}', street_id = '{$item->street_id}', streets = '{$item->streets}', house_number = '{$item->house_number}' WHERE object_id = '{$item->object_id}'");
            
            mysql_query("UPDATE objects_phones SET link_id = '{$item->link_id}' WHERE object_id = '{$item->object_id}'");
            
            mysql_query("UPDATE objects_links SET checked = '{$item->checked}' WHERE (link_id, link_s) = ('{$item->link_id}', '{$item->link_s}')") or die(mysql_error());
			
            mysql_query("UPDATE objects SET checked = '{$item->checked}' WHERE (link_id, link_s) = ('{$item->link_id}', '{$item->link_s}')") or die(mysql_error());
            
            mysql_query("INSERT INTO objects_notes (link_id, link_s, note) VALUES('{$item->link_id}','{$item->link_s}','" . mysql_real_escape_string($log) . "') ON DUPLICATE KEY UPDATE note = TRIM(CONCAT(note, '" . mysql_real_escape_string("\n" . $log) . "'))");
        }
    }
    
    public function checkPhone($phones)
    {
        $arr = [];
        
        if (!is_array($phones)) $phones = [$phones];
        foreach ($phones as $_) {
            $_ = trim($_);
            if ($_) $arr[] = $_;
        }
        if (!count($arr)) return 0;
        
        $n = mb_strlen(conf::$city->code, "utf-8") - 1;
        $code = mb_substr(conf::$city->code, 1, $n, "utf-8");
        foreach ($arr as $phone){
            /* $rt = mysql_fetch_object(mysql_query("SELECT * FROM phone_cache WHERE phone = '" . mysql_real_escape_string($phone) . "'"));
            if ($rt && $rt->is_agency) return 0;
            else if (!$rt) {
                $is_agency = 0;
                $url = "http://rent-scaner.ru/check-phone";
                $data = $then->get_url($url);
                if ($data) {
                    if (preg_match('@<meta\b(?=[^>*\bname="csrf-token"])(?=[^>*\bcontent="(.*)"])[^>]*>@isU', $data, $tmp)) {
                        $data = $this->post_url($url, false, [], ["_csrf" => $tmp[1], "CheckPhone[phone]" => $phone]);
                        if ($data && preg_match())
                    }
                }
                mysql_query("INSERT INTO phone_cache (phone, is_agency) VALUES('" . mysql_real_escape_string($phone) . "', '{$is_agency}')");
                if ($is_agency) return 0;
            } */
            
            $phone = preg_replace("/^\+7/","7",$phone);
            $tmp = mysql_real_escape_string(preg_replace("/^[78]{$code}/","",$phone));
            $phone = mysql_real_escape_string(preg_replace("/^\d/","_",$phone));
            $n = mysql_result(mysql_query("SELECT COUNT(*) FROM oris.oris_agencies WHERE phone > '' AND !deleted AND (phone = '{$tmp}' OR phone LIKE '{$phone}')"),0);
            if($n) return 0;
        }
        
        return 1;
    }
    
    public function grab($city, $deal_type, $category)
    {
        if( !isset($this->cities[$city]) ||
            !isset($this->deals[$deal_type]) ||
            !isset($this->categories[$category]) ||
            !isset($this->sections[$deal_type][$category])
        ) {
    return false;
}
        
        $this->city = $city;
        $this->deal_type = $deal_type;
        $this->category = $category;
        
        $this->session_start();
        
        $this->items = [];
        $sections = is_array($this->sections[$deal_type][$category]) ? $this->sections[$deal_type][$category] : [$this->sections[$deal_type][$category]];
        foreach($sections as $urlsfx){
            $url = $this->create_url($urlsfx);
            $pages = $this->getPages($url);
            $limit = $this->limit;
            
            for ($p = $this->startpage; $p <= $pages; $p++){
                $url = $this->create_url($urlsfx,$p);
                $items = $this->getItems($url);
                foreach($items as $item){
                    if(!$this->checkItem($item)) $limit--;
                    else{
                        $limit = $this->limit;
                        $this->items[$item->id] = $item;
                    }
                    if(!$limit) break 2;
                }
            }
        }
        
        $n = count($this->items);
        $this->__log("\n\n=========\n ITEMS FOUND: {$n}\n=========\n\n");
        $this->items = array_reverse($this->items);
        foreach($this->items as $item){
            $this->getItem($item) ? null : $this->saveUrl($item);
        }
        
        $this->session_end();

        return true;
    }
    
    public function cron()
    {
        foreach($this->cities as $city => $city_id){
            foreach($this->sections as $deal_type => $params){
                foreach($params as $category => $v){
                    $logfile = "{$_ENV[ROOT]}/../logs/{$this->site->code}-{$city}-{$deal_type}-{$category}.log";
                    /* $pidfile = "{$_ENV[ROOT]}/../pids/{$this->site->code}-{$city}-{$deal_type}-{$category}.pid";

                    if (is_file($pidfile)) {
                        $pid = file_get_contents($pidfile);
                        @shell_exec("kill -9 {$pid} > /dev/null 2>&1");
                    }

                    $cmd = "php {$_ENV[ROOT]}/grabber.php {$this->site->code} {$city} {$deal_type} {$category} >> {$logfile} 2>&1 & echo $!";
                    // echo "[",date("Y-M-d H:i:s"),"] ", $cmd ,"\n";
                    $pid = shell_exec($cmd);
                    file_put_contents($pidfile,$pid); */
                    $cmd = "php {$_ENV[ROOT]}/grabber.php {$this->site->code} {$city} {$deal_type} {$category} >> {$logfile} 2>&1 &";
                    $cmd = "flock -n /tmp/{$this->site->code}-{$deal_type}-{$category}.flock -c \"{$cmd}\"";
                    shell_exec($cmd);
                }
            }
        }
    }
    
    protected function __cron()
    {
        $logfile = "{$_ENV[ROOT]}/../logs/{$this->site->code}.log";
        $cmd = "php {$_ENV[ROOT]}/grabber.php {$this->site->code} >> {$logfile} &";
        $cmd = "flock -n /tmp/{$this->site->code}.flock -c \"{$cmd}\"";
        shell_exec($cmd);
    }

    public function get_street(&$item)
    {
        $item->street_id = 0;
        $item->streets = "";
        
        $res = [street_id => &$item->street_id, streets => &$item->streets];
        
        if(!$item->category) return $res;
        
        $re = "/\b" . preg_quote(conf::$city->name,"/") . "\b/ui";
        $address = trim(preg_replace($re,"",$item->address));
        if(mb_strlen($address,"utf-8") < 4) $address = preg_replace($re,"",$item->title);
        
        $address = preg_replace("/c/i","с",$address);
        $address = preg_replace("/ё/i","е",$address);
        $address = preg_replace("/\bлет.*\b/Uui","лет",$address);
        $address = preg_replace("/\bпереулок\b/Uui","пер",$address);
        $address = preg_replace("/\bпроспект\b/Uui","пр",$address);
        $address = preg_replace("/\bтракт\b/Uui","тр",$address);
        $address = preg_replace("/\b.*\bр(?:айо|\-о?)н\b/Uui","",$address);
        $address = preg_replace("/(\d+(?:\-я)?)/ui"," $1 ",$address);
        $address = mysql_real_escape_string($address);
                
        $__address = $item->address . PHP_EOL . $item->title;
        foreach (conf::$stopwords as $a => $id) {
            if(preg_match("/\b{$a}/isu", $__address)) {
                $item->street_id = $id;
                $item->streets = $id;
                return $res;
            }
        }
        
        $item->street_id = 0;
        $item->streets = "";
        
        $re = "/\b(ново|старо|дальне|нижне|верхне|северо|юго|южно|западно|восточно|средне)(.)/ui";
        $r = 0;
        if(preg_match($re,$address,$tmp)){
            $r = 1;
            if(preg_match("/[^а-я]/ui",$tmp[2])){
                $r = 2;
                $address = preg_replace("/\b{$tmp[1]}[^а-я]+/ui",$tmp[1],$address);
            }
        }
        
        $sql = "SELECT id, MATCH(name, prefix) AGAINST ('%s') AS rank FROM oris_streets WHERE !deleted HAVING rank > 0 ORDER BY rank DESC LIMIT 8";
        $db = mysql_query(sprintf($sql,mysql_real_escape_string($address)));
        
        if(!mysql_num_rows($db) && $r == 2){
            $address = preg_replace($re,"$1-$2",$address);
            $db = mysql_query(sprintf($sql,mysql_real_escape_string($address)));
        }
        
        $re = "/\b(?:прода.*|аренд.*|сда|сниму|квартир.*|комнат.*|адрес.*|цена|окн.*|лоджи.*|балкон|совм.*|разд.*|ремонт|договор|торг)\b/Uui";
        
        if(!mysql_num_rows($db) && $item->description){
            $tmp = preg_replace($re,"",$item->description);
            $db = mysql_query(sprintf($sql,mysql_real_escape_string($tmp)));
        }
        
        if(!mysql_num_rows($db)){
            $address = preg_replace($re,"",$address);
            $tmp = preg_split("/[^а-я]+/ui",$address);
            $address = "";
            foreach($tmp as $_) {
                $n = mb_strlen($_,"utf-8");
                if($n >= 4) $address .= mb_substr($_,0,$n - ($n < 6 ? 1 : 2),"utf-8") . "* ";
            }
            $db = mysql_query("SELECT id, MATCH(name, prefix) AGAINST ('>" . mysql_real_escape_string($address) . "' IN BOOLEAN MODE) AS rank FROM oris_streets WHERE !deleted HAVING rank > 0 ORDER BY rank DESC LIMIT 8");
        }
        
        $r = 0;
        while($_ = mysql_fetch_object($db)){
            if(!$item->street_id){
                $item->street_id = $_->id;
                $r = $_->rank;
            }
            // if($r != $_->rank) break;
            $item->streets .= $_->id . ",";
        }
        $item->streets = preg_replace("/,$/","",$item->streets);
        
        return $res;
    }
    
    public function get_house_number(&$item)
    {
        $item->house_number = "";
        $item->address = trim($item->address);
        
        if(preg_match("/\D([1-9]\d*\D{0,6})$/iu",$item->address)){
            preg_match_all("/\D(\d+)/iu",$item->address,$tmp,PREG_OFFSET_CAPTURE);
            $tmp = $tmp[1];
            $e2 = array_pop($tmp);
            while($e1 = array_pop($tmp)){
                if($e2[1] - $e1[1] > 6) break;
                $e2 = $e1;
            }
            $item->house_number = substr($item->address, $e2[1]);
            $item->house_number = preg_replace("/\\\/","/",$item->house_number);
        }
        $item->house_number = mysql_real_escape_string($item->house_number);
    }
    
    public function get_link_s($s)
    {
        return $s ? ceil(log($s/(1 - $this->dS)) / log((1 + $this->dS)/(1 - $this->dS))) : 0;
    }
    
    public static function colorize_phones($item) {
        $rx = "/^" . conf::$city->code . "/";
        $tables = [
            prodam => [
                kvartiry => "oris_flat_sale",
                komnaty => "oris_flat_sale",
                doma_dachi_kottedzhi => "oris_house_sale",
                zemelnye_uchastki => "oris_ground_sale",
                kommercheskaya_nedvizhimost => "oris_nonres_sale",
            ],

            sdam => [
                kvartiry => "oris_flat_rent",
                komnaty => "oris_flat_rent",
                doma_dachi_kottedzhi => "oris_house_rent",
                zemelnye_uchastki => "oris_ground_rent",
                kommercheskaya_nedvizhimost => "oris_nonres_rent",
            ]
        ];
        $fields = ["phone_home", "phone_work", "phone_cell"];
        
        $table = $tables[$item->deal_type][$item->category];
        if (!$table) return;
        
        $phones = [];
        foreach(explode("\n", preg_replace("/^7/sm", "8", trim($item->phone))) as $_) {
            $_ = trim($_);
            if ($_ > "" && !isset($phones[$_])) {
                $phones[$_] = [mysql_real_escape_string($_)];
                if (preg_match($rx, $_)) $phones[$_][] = mysql_real_escape_string(preg_replace($rx, "", $_));
            }
        }
        
        $s = "";
        foreach ($phones as &$_) {
            $s .= "'" . implode("','", $_) . "', ";
            $_ = "";
        }
        unset($_);
        $s = preg_replace("/,\s+$/", "", $s);
        
        $sql = "SELECT
            t1.object_id,
            t1.phone_home,
            t1.phone_work,
            t1.phone_cell,
            t1.deleted
        FROM oris.oris_objects t1
        INNER JOIN oris.{$table} t2 ON t1.object_id = t2.object_id
        WHERE
            t1.phone_home IN ({$s})
         OR t1.phone_work IN ({$s})
         OR t1.phone_cell IN ({$s})";
        $dbc = mysql_query($sql);
        while ($_ = mysql_fetch_object($dbc)) {
            foreach($fields as $f) {
                $p = $_->$f;
                if (!$p) continue;
                if (mb_strlen($p, "utf-8") < 11) $p = conf::$city->code . $p;
                if (isset($phones[$p])) $phones[$p] = $phones[$p] === "" ? $_->deleted : $phones[$p] && $_->deleted;
            }
        }
        
        foreach ($phones as $phone => $status) {
            // print "$item->object_id; $phone -> '$status'\n";
            mysql_query("UPDATE objects_phones SET status = '{$status}' WHERE object_id = {$item->object_id} AND phone = '{$phone}'");
        }
    }
    
    # TODO: rename "link" to "group", more correct
    public function link_object(&$item, $update = false)
    {
        $item->link_s = $this->get_link_s($item->s);
        $item->link_id = $item->object_id;
		
		$UpdateActualDateQuery = "UPDATE oris.GrabberObjectLinks SET ActualDate = '".$item->date."' WHERE url = '".$item->url."';";
		mysql_query($UpdateActualDateQuery) or $this->__log(mysql_error());
        
        $res = [link_id => &$item->link_id, link_s => &$item->link_s];
        
        // if (!$item->category || $item->category != "kvartiry" && $item->category != "komnaty") {
            // self::colorize_phones($item);
            // return $res;
        // }
        
        if($item->street_id && $item->phone != "Нет телефона"){
            # TODO: review this
            # use link_s in search, make link_id as single key whitout link_s
            		$filter = "";
			
			if($item->category == 'doma_dachi_kottedzhi'){
				$filter = " AND o.s IS NOT NULL AND o.s > 0 AND ABS(o.s - {$item->s}) < 1 ";
			}else
				
			if($item->category == 'zemelnye_uchastki'){
				if($item->s && $item->s > 0){
				$filter = " AND o.s IS NOT NULL AND o.s > 0 AND ABS(o.s - {$item->s}) < 1 ";
					
				}else if($item->land && $item->land > 0){
				$filter = " AND o.land IS NOT NULL AND o.land > 0 AND ABS(o.land - {$item->land}) < 1 ";
				}
				else{
					$filter = " AND 1!=1 ";
					}
			}
			
			$query  = "
                SELECT o.link_id FROM objects o
                INNER JOIN objects_phones p ON o.object_id = p.object_id
                INNER JOIN objects_phones t ON p.phone = t.phone AND p.object_id != t.object_id
                WHERE
                    o.display
                AND o.link_id
                AND t.object_id = '{$item->object_id}'
                AND o.category  = '{$item->category}'
                AND o.deal_type = '{$item->deal_type}'
                AND o.street_id = '{$item->street_id}'
                AND o.level = '{$item->level}'
                AND o.rooms = '{$item->rooms}'
					{$filter}
                LIMIT 1;
            ";
            $tmp = mysql_fetch_assoc(mysql_query($query));
			
            if($tmp["link_id"]) $item->link_id = $tmp["link_id"];
        }
        
        $link = new stdClass();
        foreach(["date","date_add","link_id","link_s","street_id","level","rooms","deal_type","category","is_agency"] as $f) $link->$f = $item->$f;
        $link->url = "";
        $link->checked = 0;
        $link->number = "0";
        $link->phone = [];
        $link->price = [];
        $link->images = [];
        $link->sites = [];
        $link->is_delayed = 0;
        $tmp = [
            images => 0,
            url => 0,
        ];
        $code = conf::$city->code;
        $re = "/^" . $code . "/";
        
        $streets = explode(",", $item->streets);
        $streets = array_combine($streets, $streets);
        
        # TODO: review algorythm
        # select $link from `objects_links` first, then compare with $item (see previous todo)
        # so next loop can be removed
        # (???) races with other process -> rebuild `object_links` with another unique proccess/cron, NOT HERE!! -> so grabbers could work faster!!
        # store combined streets of linked objects in `objects_links`
        $dbc = mysql_query("SELECT * FROM objects WHERE object_id = '{$item->object_id}' OR link_id = '{$item->link_id}' AND link_s = '{$item->link_s}' ORDER BY date DESC, date_add DESC");
        $link->count = mysql_num_rows($dbc);
		
        $checked = 0;
        while ($_ = mysql_fetch_object($dbc)) {
            foreach (["title","raion","address","distance","params","seller","is_agency"] as $f) if (mb_strlen($link->$f,"utf-8") < mb_strlen($_->$f,"utf-8")) $link->$f = $_->$f;
            if($_->date > $link->date) $link->date = $_->date;
            $link->sites[$_->site_id] = $this->sites[$_->site_id]->name;
            
            $_->phone = explode("\n",preg_replace("/^7/sm", "8", trim($_->phone)));
            foreach($_->phone as $f) {
                $f = trim($f);
                if($f > "" && !isset($link->phone[$f])){
                    $link->phone[$f] = [mysql_real_escape_string($f)];
                    if(preg_match($re, $f)) $link->phone[$f][] = mysql_real_escape_string(preg_replace($re, "", $f));
                }
            }
            
            if(!count($link->images) && trim($_->images) > ""){
                $_->images = preg_split("/(?:\r?\n){1,}/", trim($_->images));
                $tmp["images"] = $_->site_id;
                $link->images = $_->images;
            }
            
            if(!$link->url && trim($_->url) > ""){
                $tmp["url"] = $_->site_id;
                $link->url = trim($_->url);
            }
            
            if(!$link->description && trim($_->description) > "") {
                $link->description = trim($_->description);
            }
            
            if(!count($link->price)) $link->price = [$_->s => [$_->price]];
            
            if($_->checked && !$checked) $checked = $_->checked;
        }
        
        # images
        $host = $this->sites[$tmp["images"]]->host;
        $host = "http" . (preg_match("/[:]443$/", $host) ? "s" : "") . "://" . $host;
        foreach($link->images as &$_){
            if(!preg_match("/^https?\:\/\//is", $_)) $_ = $host . $_;
        }
        unset($_);
        $link->images = implode("\n", $link->images);
        
        # url
        if(!preg_match("/^https?\:\/\//is",$link->url)){
            $host = $this->sites[$tmp["url"]]->host;
            $host = "http" . (preg_match("/[:]443$/",$host) ? "s" : "") . "://" . $host;
            $link->url = $host . $link->url;
        }
        
        $link->sites = implode(", ",$link->sites);
        
        # prices
        /*foreach($link->price as &$_) {
            $_ = array_values($_);
            sort($_, SORT_STRING);
        }
        unset($_);
        ksort($link->price, SORT_NUMERIC);*/
        
        # phones
        $tmp = "";
        foreach($link->phone as &$_){
            $tmp .= "'" . implode("','",$_) . "', ";
            $_ = "";
        }
        unset($_);
        $tmp = preg_replace("/,\s+$/","",$tmp);
		$allItemPhonesSqlInConcated = $tmp;
        $dbc = mysql_query("
            SELECT
                oo.date_check,
                oo.object_id,
                oo.square,
                oo.rooms_count,
                oo.level,
                oo.price,
                oo.phone_home,
                oo.phone_work,
                oo.phone_cell,
                oo.deleted,
				COALESCE((SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$link->url}'  LIMIT 1),'') = oo.object_id as haveSavedUrl,
                oo.street_id,
                oo.number,
                IF(
					ofs.object_id IS NULL AND ohs.object_id IS NULL AND ogs.object_id IS NULL AND ons.object_id IS NULL,
					IF(ofr.object_id IS NULL AND ohr.object_id IS NULL AND ogr.object_id IS NULL AND onr.object_id IS NULL, NULL, 'sdam'), 
                    'prodam') AS deal_type,
				CASE
					WHEN 
						ofs.object_id IS NOT NULL 
                        OR ofr.object_id IS NOT NULL 
                        AND LOWER((SELECT name FROM oris.oris_series_type s WHERE s.id = (SELECT series_id FROM oris.oris_flat WHERE object_id = oo.object_id))) NOT LIKE '%комната%' 
					THEN 'kvartiry'
					WHEN 
						ofs.object_id IS NOT NULL 
						OR ofr.object_id IS NOT NULL 
                        AND LOWER((SELECT name FROM oris.oris_series_type s WHERE s.id = (SELECT series_id FROM oris.oris_flat WHERE object_id = oo.object_id))) LIKE '%комната%' 
					THEN 'komnaty'
					WHEN ogs.object_id IS NOT NULL OR ogr.object_id IS NOT NULL THEN 'zemelnye_uchastki'
					WHEN ohs.object_id IS NOT NULL OR ohr.object_id IS NOT NULL THEN 'doma_dachi_kottedzhi'
					WHEN ons.object_id IS NOT NULL OR onr.object_id IS NOT NULL THEN 'kommercheskaya_nedvizhimost'
				END as category
            FROM oris.oris_objects oo
            LEFT JOIN oris.oris_flat_sale ofs ON oo.object_id = ofs.object_id
            LEFT JOIN oris.oris_flat_rent ofr ON oo.object_id = ofr.object_id
            LEFT JOIN oris.oris_house_sale ohs ON oo.object_id = ohs.object_id
            LEFT JOIN oris.oris_house_rent ohr ON oo.object_id = ohr.object_id
            LEFT JOIN oris.oris_ground_sale ogs ON oo.object_id = ogs.object_id
            LEFT JOIN oris.oris_ground_rent ogr ON oo.object_id = ogr.object_id
            LEFT JOIN oris.oris_nonres_sale ons ON oo.object_id = ons.object_id
            LEFT JOIN oris.oris_nonres_rent onr ON oo.object_id = onr.object_id
            WHERE
                oo.phone_home IN ({$tmp})
             OR oo.phone_work IN ({$tmp})
             OR oo.phone_cell IN ({$tmp})
             OR oo.object_id = (SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$link->url}' LIMIT 1)
            ORDER BY
                oo.date_check DESC
        ");
        
        $tmp = ["phone_home", "phone_work", "phone_cell"];
        $__cache = [];
        $__deal_type = [];
        $__category = [];
		
        unset($link->same, $link->archive, $__price);
        while($_ = mysql_fetch_object($dbc)){
            $__deal_type[$_->deal_type] = 1;
            $__category[$_->category] = 1;
            if ((isset($streets[$_->street_id]) && $link->deal_type == $_->deal_type && $link->category == $_->category)) {
                $_->link_s = $this->get_link_s($_->square);
                if (!isset($link->same) || $link->same == 255) {
                    $link->same = $_->deleted ? 255 : 1;
                }
                
                if (((is_null($link->level) || $link->level == $_->level) &&
                    (is_null($link->rooms) || $link->rooms == $_->rooms_count )&&
                    ($link->link_s == $_->link_s || abs($item->s - $_->square) <= 1))
                ) {
                    $link->street_id = $_->street_id;
                    $link->archive = isset($link->archive) ? $link->archive && $_->deleted : $_->deleted;
                    if ($link->number === "0") $link->number = $_->number;
                    if (!$_->deleted && !isset($__price)) $__price = $_->price;
                    
                    foreach($tmp as $f) {
                        $f = $_->$f;
                        if (!$f) continue;
                        if (mb_strlen($f, "utf-8") < 11) $f = $code . $f;
                        if (isset($link->phone[$f])) {
                            $link->phone[$f] = $link->phone[$f] === "" ? $_->deleted : $link->phone[$f] && $_->deleted;
                            $__cache[$f] = 1;
                        }
                    }
                    
                    continue;
                }
            }
			
            
            foreach($tmp as $f) {
                $f = $_->$f;
                if(!$f) continue;
                if(mb_strlen($f, "utf-8") < 11) $f = $code . $f;
                if(!isset($__cache[$f]) && isset($link->phone[$f])) $link->phone[$f] = $link->phone[$f] === "" ? $_->deleted : $link->phone[$f] && $_->deleted;
            }
        }
        $dbc = mysql_query("SELECT 1 as result FROM oris.GrabberObjectLinks WHERE Url LIKE '%{$item->url}%' AND ObjectId IN (SELECT object_id FROM oris.oris_objects WHERE !deleted) LIMIT 1;");
		$haveSavedUrl = mysql_fetch_object($dbc);
		$haveSavedUrl = $haveSavedUrl->result;
		
		if($haveSavedUrl && strpos($item->url, 'avito') !== false)
		{
			$isPhoneAvitoBlocked = strpos($item->description, 'Номер продавца защищён Avito') !== false;
			$que = mysql_query("
SELECT SUM(b) as result FROM (
SELECT (SELECT COALESCE(phone_home,'') as phone FROM oris.oris_objects WHERE object_id = (SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$item->url}')) IN ({$allItemPhonesSqlInConcated}) as b
UNION
SELECT (SELECT COALESCE(phone_work,'') as phone FROM oris.oris_objects WHERE object_id = (SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$item->url}')) IN ({$allItemPhonesSqlInConcated}) as b
UNION
SELECT (SELECT COALESCE(phone_cell,'') as phone FROM oris.oris_objects WHERE object_id = (SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$item->url}')) IN ({$allItemPhonesSqlInConcated}) as b
) as l
;");
			$isAnyPhoneInObject = mysql_fetch_object($que);
			$isAnyPhoneInObject = $isAnyPhoneInObject->result;
						
			$haveSavedUrl = $haveSavedUrl && ($isPhoneAvitoBlocked || (!$isPhoneAvitoBlocked && $isAnyPhoneInObject));
		}
		
        # проверяем отсутствие телефона в категории + признак архива
        $__f = !isset($__deal_type[$link->deal_type]) || !isset($__category[$link->category])  || $link->archive;
        
        # проверяем наличие новых и архивных телефонов
        if (!$__f) foreach ($link->phone as $_) {
            if ($_ === "" || $_ === "1" || $_ === true) {
                $__f = 1;
                break;
            }
        }
        
		$update_backup = $update;
        # объявление - неархивный "полный дубль" без новых телефонов c ссылкой
        // if (!$__f && !$link->archive && $link->number !== "0" && isset($__price)) {						
        if (!$__f && !$link->archive && ((($link->category == "komnaty" || $link->category == "kvartiry") && $link->number !== "0") || ($link->category !== "kvartiry" && $link->category !== "komnaty"))) {
            # сравниваем цены
            // $price = __floatval(current(current($link->price)));
            // $t = [];
            // $t[$price] = 1;
            // $t[$price / 1000] = 1;
            // $t[$price * 1000] = 1;
			// if(isset($t[$__price]))
				// $link->checked = 1;
			// else 
			if($haveSavedUrl)
				$link->checked = 2;	
        }
        else if (!$__f || ($haveSavedUrl)) {
            # повторное получение объявления сайта
            if ($update && isset($this->olditems[$item->id])) {
                $olditem = $this->olditems[$item->id];
                $diff = ['sdam' => 30, 'prodam' => 90];
                // if ($checked && ($olditem->date_diff >= $diff[$item->deal_type] || $update['price'])) {
                if ($checked && $olditem->date_diff && ($haveSavedUrl)) {
                    $checked = 2;
                }
                
                unset($update['date'], $update['price'], $update['levels'], $update['s']);
                foreach($update as $_) {
                    if($_) {
                        $checked = 0;
                        break;
                    }
                }
            }
            
            $link->checked = $checked;
        }
		
		
		$que = mysql_query("SELECT checked FROM grabber.objects_links WHERE (link_id, link_s) = ('{$link->link_id}', '{$link->link_s}');");
		$currentCheckedStatus = mysql_fetch_object($que);
		$currentCheckedStatus = $currentCheckedStatus->checked;
		
		$urlContainsCian = strpos($item->url, 'cian.ru') !== false;
		$urlContainsIrr = strpos($item->url, 'irr.ru') !== false;
		$urlContainsDomofond = strpos($item->url, 'domofond.ru') !== false;
		
		$excludeAutoUpdate = $urlContainsCian || $urlContainsIrr || $urlContainsDomofond;
		
		if(!$excludeAutoUpdate && $link->checked == 2 && $currentCheckedStatus != 2)
			{
				$doesOnlyDateOrPriceChanged = $update_backup['date'] || $update_backup['price'];
				unset($update_backup['date'], $update_backup['price']);
				foreach($update_backup as $_) 
				{
                    if($_) 
					{
                        $doesOnlyDateOrPriceChanged = 0;
                        break;
					}
				}

				if($doesOnlyDateOrPriceChanged)
				{
					$que = mysql_query("SELECT ObjectId FROM oris.GrabberObjectLinks WHERE Url = '{$item->url}';");
					$res = mysql_fetch_object($que);
					$objectId = $res->ObjectId;
					$que = mysql_query("SELECT date_check,IF(object_id IN (
SELECT object_id FROM oris.oris_flat_sale
UNION ALL
SELECT object_id FROM oris.oris_house_sale
UNION ALL
SELECT object_id FROM oris.oris_ground_sale
UNION ALL
SELECT object_id FROM oris.oris_nonres_sale
),price*1000,price) as price, is_on_handsafe,private_note  FROM oris.oris_objects WHERE object_id = '{$objectId}';");
					$res = mysql_fetch_object($que);
					$oldDateCheck = $res->date_check;
					$oldPrice = $res->price;
					$oldIsOnHandsafe = $res->is_on_handsafe;
					$oldPrivateNote = $res->private_note;
					$que = mysql_query("SELECT id FROM oris.oris_users_v3 WHERE login = 'Robot';");
					$res = mysql_fetch_object($que);
					$robotUserId = $res->id;
					$que = mysql_query("SELECT NOW() as date2;");
					$res = mysql_fetch_object($que);
					$fixedNow = $res->date2;
					
					if($objectId != null && $oldDateCheck != null && $oldPrice != null && $robotUserId != null && $fixedNow != null)
					{
						$currentPrice = __floatval(current(current($link->price)));
						
						$handsafeRegexReplace = "/((,)*( )*((н|Н)а задатке до)+( | )?([0-9]?[0-9]?(\ |\ )??((я|Я)нвар|(ф|Ф)еврал|(м|М)арт|(а|А)прел|(м|М)а|(и|И)юн|(и|И)юл|(а|А)август|(с|С)ентябр|(о|О)ктябр|(н|Н)оябр|(д|Д)екабр)(ь|я|а|й)?(\ |\ )?[0-9]?[0-9]?[0-9]?[0-9]?)?[\ \,]*\s*)|((,)*( )*((с|С)няли с продажи до)+( | )?([0-9]?[0-9]?(\ |\ )??((я|Я)нвар|(ф|Ф)еврал|(м|М)арт|(а|А)прел|(м|М)а|(и|И)юн|(и|И)юл|(а|А)август|(с|С)ентябр|(о|О)ктябр|(н|Н)оябр|(д|Д)екабр)(ь|я|а|й)?(\ |\ )?[0-9]?[0-9]?[0-9]?[0-9]?)?[\ \,]*\s*)|((,)*( )*((с|С)няли с продажи на неопределенный срок)+[\ \,]*\s*)|((,)*( )*((е|Е)сть потенциальный покупатель)+[\ \,]*\s*)/smi";
						
						
						$changesToOrisObject = " date_check = '{$fixedNow}' ";
						$abortChanges = false;
						
						
						if($currentPrice != $oldPrice)
						{
							if(($link->deal_type == 'prodam' && abs($currentPrice - $oldPrice) <= 100000) || ($link->deal_type == 'sdam' && abs($currentPrice - $oldPrice) <= 1000))
							{
								$priceToApply = $link->deal_type == 'prodam' ? $currentPrice / 1000 : $currentPrice;
								$oldPrice = $link->deal_type == 'prodam' ? $oldPrice / 1000 : $oldPrice;
								$changesToOrisObject = $changesToOrisObject . " , price = {$priceToApply} ";
							}
							else $abortChanges = true;
						}
						
						if($oldIsOnHandsafe == 1)
						{
							$changesToOrisObject = $changesToOrisObject . " , is_on_handsafe = 0 ";
							
							$newMemo = preg_replace($handsafeRegexReplace,'',$oldPrivateNote);
							if($newMemo !== $oldPrivateNote)
							{
								$changesToOrisObject = $changesToOrisObject . " , private_note = \"{$newMemo}\" ";							
							}
						}
						
						if(!$abortChanges)
						{
							$this->__log($objectId);
							$this->__log($changesToOrisObject);
							$res = mysql_query("INSERT INTO `oris`.`oris_changes` (`object_id`, `date`, `table_name`, `user_id`, `flag`) VALUES ('{$objectId}', '{$fixedNow}', 'oris_objects', {$robotUserId}, 1);");
							// $id = mysql_insert_id();
							$res = mysql_query("UPDATE oris.oris_objects SET {$changesToOrisObject} WHERE object_id = '{$objectId}';");
							$link->checked = 1;
							$res = mysql_query("UPDATE `oris`.`GrabberObjectLinks` SET `ActualDate` = NOW(), `DeleteDate` = NULL, `ExpiredDate` = NULL, `InspectorLastCheckDate` = NOW() WHERE `Url` = '{$item->url}';");
							
							$que = mysql_query("SELECT Id FROM oris.GrabberObjectLinks WHERE url = '{$item->url}'");
							$result = mysql_fetch_object($que);
							$urlId = $result->Id;							
							
							$res = mysql_query("DELETE FROM oris.oris_incorrect_objects_request WHERE object_id = '{$objectId}';");
							if($res && mysql_affected_rows() > 0)
							{
								$que = mysql_query("SELECT memo FROM oris.oris_objects WHERE object_id = '{$objectId}';");
								$result = mysql_fetch_object($que);
								$memo = $result->memo;	
								if(empty(memo) || is_null(memo))
									$newMemo = 'Удалено из некорректного';
								else
									$newMemo = $memo."
Удалено из некорректного";


								$changesToOrisObject = " memo = \"{$newMemo}\" ";
								$this->__log($objectId);
								$this->__log($changesToOrisObject);
								$res = mysql_query("INSERT INTO `oris`.`oris_changes` (`object_id`, `date`, `table_name`, `user_id`, `flag`) VALUES ('{$objectId}', '{$fixedNow}', 'oris_objects', {$robotUserId}, 1);");
								$res = mysql_query("UPDATE oris.oris_objects SET {$changesToOrisObject} WHERE object_id = '{$objectId}';");
							}
							$res = mysql_query("DELETE FROM oris.GrabberObjectLinkBadRequests WHERE GrabberObjectLinkId = {$urlId};");							
						}
					}
				}
			}
		
        
        if(!$update && $link->checked) {
            mysql_query("UPDATE objects SET checked = '{$link->checked}' WHERE object_id = '{$item->object_id}' OR (link_id, link_s) = ('{$item->link_id}', '{$item->link_s}')") or die(mysql_error());
        }
        
        $item->checked = $link->checked;
        
        $link->phone = serialize($link->phone);
        $link->price = serialize($link->price);
        
        $data = (array) $link;
        unset($data["city"]);
        // unset($data["category"]);
        foreach ($data as &$_) $_ = mysql_real_escape_string($_);
        unset($_);
        
        $sql = "REPLACE INTO objects_links (`" . implode("`, `", array_keys($data)) . "`) VALUES('" . implode("', '", $data) . "')";
        mysql_query($sql) or $this->error(mysql_error());
        
        return $res;
    }
	
	public function link_unlinked(){		
		$this->__log('Started link_unlinked');
		 $dbc = mysql_query("
            SELECT COUNT(*) as cnt
			FROM grabber.objects 
			WHERE 
				(link_id,link_s) NOT IN (SELECT link_id,link_s FROM grabber.objects_links)  
				AND !(display = 0 AND checked = 0);
        ");
		$cnt = mysql_fetch_object($dbc);
		$this->__log('');
		$this->__log('-------------------------------');
		$this->__log("Founded unlinked items - {$cnt->cnt}");
		$this->__log('-------------------------------');
		 $dbc = mysql_query("
            SELECT * 
			FROM grabber.objects 
			WHERE 
				(link_id,link_s) NOT IN (SELECT link_id,link_s FROM grabber.objects_links)  
				AND !(display = 0 AND checked = 0);
        ");
		
        while($item = mysql_fetch_object($dbc)){
			
			
			$this->__log("({$item->link_id},{$item->link_s}) - {$item->url}");
			$this->link_object($item);
			
                
            mysql_query("UPDATE objects SET link_id = '{$item->link_id}', link_s = '{$item->link_s}', street_id = '{$item->street_id}', streets = '{$item->streets}', house_number = '{$item->house_number}' WHERE object_id = '{$item->object_id}'");
            
            mysql_query("UPDATE objects_phones SET link_id = '{$item->link_id}' WHERE object_id = '{$item->object_id}'");
		}
	}

    public function saveUrl($item)
    {
        $item->site_id = $this->site->id;
        $item->url = mysql_real_escape_string($item->url);

        if(!$item->city) $item->city = $this->city;
        if(!$item->deal_type) $item->deal_type = $this->deal_type;
        if(!$item->category) $item->category = $this->category;
        
        mysql_query("INSERT INTO objects_errors (site_id,id,url,date_add,date_check,fails,city,deal_type,category) VALUES('{$item->site_id}','{$item->id}','{$item->url}',NOW(),NOW(),0,'{$item->city}','{$item->deal_type}','{$item->category}') ON DUPLICATE KEY UPDATE fails = fails + 1, date_check = NOW()") or $this->__log("saveUrl(): " . mysql_error());
    }

    public static function parse_level($s)
    {
        $result = [];
        $mathes = [];
        $count = 0;
        $patterns = [
            # 2-х этажный
            '/\b(\d+)\D{0,4}\bэт(?:\b|аж)/ismu',
            
            # кол-во этажей 5
            '/\bэтаж(?:ей?|ность)?\b\D{0,4}(\d+)\b/ismu',
            
            # Этаж 4 из 16, Этаж 4/17
            '/\bэтаж\b\D{1,4}\b\d+\s*(?:\bиз\b|\/)\s*(\d+)\b/ismu'
        ];
        
        foreach($patterns as $rx){
            $r = preg_match_all($rx,$s,$tmp);
            if($r){
                $mathes = array_merge($mathes, $tmp[1]);
                $count += $r;
            }
        }
        
        # 1-к вартиру, 6/18 30/12/7
        if(!$count && $r = preg_match_all('/\b(\d+)[^\d\/]{0,3}\/[^\d\/]{0,3}(\d+)[^\d\/]{3,}/ismu',$s . "___",$tmp)){
            for($i = 0; $i < $r; $i++){
                if($tmp[1][$i] <= $tmp[2][$i]){
                    $mathes[] = $tmp[1][$i];
                    $count++;
                }
            }
        }
        
        return $count ? min($mathes) : 0;
    }
    
    public static function parse_levels($s)
    {
        $mathes = [];
        $count = 0;
        $patterns = [
            # 2-х этажный/уровневый
            '/\b(\d+)\D{0,4}\b(?:эт(?:\b|аж)|уров)/ismu', 
            
            # кол-во этажей/уровней 5; этажей в здании
            '/
                \b(?:этаж(?:ей?|ность)?|уровн?е(?:нь|й|вый)?)\b
                (?:\s+в\s+(?:доме|здании|помещении|коттедже))?
                \D{0,4}
                (\d+)\b
            /ismux',
            
            # Этаж 4 из 16, Этаж 4/17
            '/\bэтаж\b\D{1,4}\b\d+\s*(?:\bиз\b|\/)\s*(\d+)\b/ismu'
        ];
        
        foreach($patterns as $rx){
            $r = preg_match_all($rx,$s,$tmp);
            if($r){
                $mathes = array_merge($mathes, $tmp[1]);
                $count += $r;
            }
        }
        
        # 1-к вартиру, 6/18 30/12/7
        if(!$count && $r = preg_match_all('/\b(\d+)[^\d\/]{0,3}\/[^\d\/]{0,3}(\d+)[^\d\/]{3,}/ismu',$s . "___",$tmp)){
            for($i = 0; $i < $r; $i++){
                if($tmp[1][$i] <= $tmp[2][$i]){
                    $mathes[] = $tmp[2][$i];
                    $count++;
                }
            }
        }
        
        return $count ? max($mathes) : 0;
    }

    public static function parse_square($s)
    {
        $result = [];
        $mathes = [];
        $count = 0;
        
        $patterns = [
            '/
                \b(\d+(?:[,.]\d+)?)
                \s*\/\s*
                (?:\d+(?:[,.]\d+)?|-)
                \s*\/\s*
                (?:\d+(?:[,.]\d+)?|-)\b
            /ismx',
            '/
                \b(\d+(?:[.,]\d+)?)
                \D{0,3}
                (?:кв[.\s]\s*м|м\s*[2²]\b|квадрат\w{0,2})
            /ismux',
            '/
                \bпл(?:ощадь)?\b
                (?:\s+(?:общая|жилая|дома|квартиры|комнаты|коттеджа|офиса|помещения|здания)\b)?
                [\s.,:;\(\)]+
                (?:в\s*)?
                (?:кв[.\s]\s*м|м\s*[2²]\b)
                [\s.,:;\(\)]+
                (\d+(?:[,.]\d+)?)
            /ismux',
            '/
                \b(?:комнат[ау]|квартир[ау]|дом|помещение|офис)\b
                [\s.,:;\(\)]+
                (\d+(?:[,.]\d+)?)
                \D{0,3}м
            /ismux',
        ];
        
        foreach($patterns as $rx){
            $r = preg_match_all($rx,$s,$tmp);
            if($r){
                foreach($tmp[1] as &$_) $_ = __floatval($_);
                unset($_);
                
                $mathes = array_merge($mathes, $tmp[1]);
                $count += $r;
            }
        }
        
        return $count ? max($mathes) : 0;
    }

    public static function parse_land($s)
    {
        $result = [];
        $mathes = [];
        $count = 0;

        $patterns = [
            '/\b(\d+(?:[,.]\d+)?)\b\D{0,4}\b(?:сот|га|гектар)/ismu',
            '/
                \b(?:участ(?:ок|к[ае])|земл[иея])
                [\s.,:;\(\)]+
                (?:в\s*)?
                (?:сот(?:ок|ках)?|га|гектар(?:ах)?)
                [\s.,:;\(\)]+
                (\d+(?:[,.]\d+)?)
            /ismux',
        ];
        
        foreach($patterns as $rx){
            $r = preg_match_all($rx,$s,$tmp);
            if($r){
                foreach($tmp[1] as &$_) $_ = __floatval($_);
                unset($_);
                
                $mathes = array_merge($mathes, $tmp[1]);
                $count += $r;
            }
        }
        
        return $count ? max($mathes) : 0;
    }
    
    public static function parse_rooms($s, $c = "")
    {
        $arr = [
            "одн" => 1,
            "дву" => 2,
            "тр"  => 3,
            "четыр" => 4,
            "пяти"  => 5,
            "шести" => 6,
        ];
        if(preg_match('/
            \b(
                (?:одн[уеё]|дву|тр[её]|четыр[её])шк
                |
                (?:одно?|двух?|тр[её]х?|четыр[её]х?|пяти?|шести?)\W{0,3}комн
            )
        /ismxu', $s, $tmp)) foreach($arr as $k => $v) if(strpos($tmp[1],$k) !== false) return $v;
        if(preg_match('/\b(\d)\s*[\D\S]{0,3}\s*к(?:ом(?:н(?:ат\w{0,3})?)?)?\b/ismu', $s, $tmp)) return $tmp[1];
        if(preg_match('/\bкомн\w{0,2}\D{0,3}(\d+)\D/ismu', $s, $tmp)) return $tmp[1];
        return ($c == "kvartiry" || $c == "komnaty" ? 1 : 0);
    }
    
    public function isValidItemHtml($html, $item)
    {
        return $this->error("isValidItemHtml: pseudo abstract method, must be overriden");
    }

    protected function get_mirror($deleteCurrent = false)
    {
        if(!count($this->mirrors)) return $this->error("get_mirror: mirror list is empty");
        $mirror = $this->mirror ? next($this->mirrors) : current($this->mirrors);
        if(!$mirror){
            reset($this->mirrors);
            $mirror = current($this->mirrors);
        }
        
        if($deleteCurrent){
            $this->__log("deleting mirror: {$this->mirror}");
            reset($this->mirrors);
            foreach($this->mirrors as $k => $v){
                if($v == $this->mirror){
                    unset($this->mirrors[$k]);
                    break;
                }
            }
        }
        
        $this->mirror = $mirror;
        return $this->mirror;
    }

    protected static function url_get_contents($url, $params = [], $method = "GET")
    {
        if($method == "GET") {
            if(is_array($params) && count($params)) $url = $url . "?" . http_build_query($params);
            echo "[" . date("Y-m-d H:i:s") . "] url_get_contents: {$url}\n";
            
            $context = stream_context_create([ http => [
                "protocol_version" => 1.1,
                "method"  => "GET",
                "timeout" => 60,
                "header" => [ "Connection: close" ],
            ]]);
        }
        elseif($method == "POST") {
            echo "[" . date("Y-m-d H:i:s") . "] url_get_contents: {$url}\n" . print_r($params, 1);
            
            $content = http_build_query($params);
            $context = stream_context_create([ http => [
                "protocol_version" => 1.1,
                "method"  => "POST",
                "timeout" => 60,
                "header" => [
                    "Content-Type: application/x-www-form-urlencoded",
                    "Content-Length: " . strlen($content),
                    "Connection: close",
                ],
                "content" => $content,
            ]]);
        }
        else {
            die ("[" . date("Y-m-d H:i:s") . "] Unknown request method: \"{$method}\"\n");
        }
        
        $data = file_get_contents($url, false, $context);
        sleep(3);
        return $data;
    }

    protected function mirror_get_contents($params, $method = "GET")
    {
        $this->get_mirror();
        
        if(!is_array($params)) {
            if(preg_match("@^\d+$@", $params)) $params = [item_id => $params];
            else if(preg_match("@^https?://@is", $params)) $params = [url => $params];
            else $params = [];
        }
        
        $data =  @gzdecode(self::url_get_contents($this->mirror, $params, $method));
        if(!$data) {
            $this->__log($this->mirror . " failed, trying next...");
            $this->get_mirror(true);
            return $this->mirror_get_contents($params, $method);
        }
        return $data;
    }

    protected function get_url_head($url, $proxy = false, $opts = [])
    {
        $headers = [];
        $headers = is_array($opts) ? $opts : [$opts];
        $headers[] = "Accept-Language: ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3";
        $headers[] = "Cache-Control: max-age=0";
        $headers[] = "Connection: keep-alive";
        $headers[] = "User-Agent: Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0";
        
        $curl = curl_init();
        
        if($proxy){
            $this->get_proxy($proxy === true ? "" : $proxy);
            curl_setopt($curl, CURLOPT_PROXY, $this->proxy);			
            if ($this->__proxy->pass) curl_setopt($curl, CURLOPT_PROXYUSERPWD, $this->__proxy->login . ":" . preg_replace("/\s/","",$this->__proxy->pass));
            // $headers[] = "X-Forwarded-For: {$this->__proxy->ip}";
        }
        else{
            $this->__proxy = null;
            $this->proxy = null;
            sleep(3);
        }
        
        curl_setopt($curl, CURLOPT_HTTPHEADER, $headers);
        curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout);
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_NOBODY, true);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
        curl_exec($curl);
        $info = curl_getinfo($curl);
        curl_close($curl);
        
        return $info;
    }

    public function setSiteCodeAlias($code){
        $this->site->code_alias = $code;
    }
}