deals = [ prodam => "prodam", sdam => "arenda/sdam", ]; $this->sections = [ prodam => [ kvartiry => "kvartiry", doma_dachi_kottedzhi => "doma", zemelnye_uchastki => ["dachi", "zemlya"], kommercheskaya_nedvizhimost => "commercial", ], // sdam => [ // kvartiry => "dolgosrochno/kvartiry", // komnaty => "dolgosrochno/komnaty", // kommercheskaya_nedvizhimost => "commercial", // ] ]; $this->datelimit = date("Y-m-d", time() - 60 * 86400) . " 00:00:00"; $this->check_olditems = true; } public function getPages($url, $n = 0) { if($n){ $err = "getPages({$url}) - error receiving data"; $this->update_proxy(1, -1, $err); if($n > $this->fails) return $this->__log($err); //записать ошибку в лог } $this->data = $this->get_url($url, true); /* $file = $_ENV["ROOT"] . "/~tmp/" . $this->site->code . "-" . $this->deal_type . "-" . $this->category . ".html"; if (!is_file($file)) { $this->data = $this->get_url($url, false); file_put_contents($file, $this->data); } else { $this->data = file_get_contents($file); } */ $r = preg_match('@]+\bclass="[^"]*\bpaginator__pages\b[^"]*">(.*)@isU', $this->data, $tmp); if (!$r) return 1; $r = preg_match_all('@]*>\s*(\d+)\s*@isU', $tmp[1], $tmp); if (!$r) return 1; $p = max($tmp[1]); return $p; } public function getItems($url, $n = 0) { if($n){ $err = "getItems({$url}) - error receiving data"; $this->update_proxy(1, -1, $err); if($n > $this->fails){ $this->__log($err); return []; } } $data = $this->data ? $this->data : $this->get_url($url, true); $this->data = null; $tmp = preg_split('@]*>\s*\bblockFotoramaData\b[^;]*\;\s*\bvar\b\s*\brawData\b\s*\=\s*@is', $data); $tmp = preg_split('@\;\s*\bvar\b\s*\bpageMeta\b@is', $tmp[1]); $tmp = trim($tmp[0]); $json = false; $titles = []; if ($tmp > "") { $json = json_decode($tmp); $r = preg_match_all('@]+\bhref="/view/(\d+)/[^"]*"[^>]*>(.*)@isU', $data, $tmp); for ($i = 0; $i < $r; $i++) { if ($titles[$tmp[1][$i]] > "") continue; $titles[$tmp[1][$i]] = trim(html_entity_decode(strip_tags($tmp[2][$i]))); } } if (!$json || !isset($json->services)) return $this->getItems($url, $n + 1); $__params = [ "reference_point" => "Ориентир", "house_material_type" => "Материал дома", "type" => "Тип здания", "layout_type" => "Планировка", "floor" => "Этаж", "floors_count" => "Этажность", "wc_type" => "Санузел", "rooms_count" => "Комнат", "total_area" => "Общая площадь", "living_area" => "Жилая площадь", "kitchen_area" => "Кухня", "condition_type" => "Состояние", "balcony_count" => "Балконов", "loggia_count" => "Лоджий", "has_phone" => "Телефон", "has_internet" => "Интернет", "parking_type" => "Парковка", "ownership_type" => "Форма собственности", "is_exchange" => "Обмен", "is_mortgage" => "Ипотека", "is_pawn" => "В залоге", "is_deposit" => "Внесен задаток", "is_haggle" => "Торг", "is_net_sale" => "Чистая продажа", "is_newbuilding" => "Новостройка", // "region" => "Область", // "city" => "Город", "district" => "Район", "microdistrict" => "Микрорайон", "street" => "Улица", "house_number" => "Номер дома", "release_date" => "Год постройки", // "released" => "Дом сдан", "builder" => "Застройщик", "from_builder" => "От застройщика", // "project_declaration" => "Проектная декларация", "house_type" => "Тип дома", "house_class" => "Класс дома", "metro" => "Метро", "area" => "", "homestead_area" => "Площадь участка", "gas_supply_type" => "Газоснабжение", "has_electricity" => "Электроснабжение", "heating_location" => "Отопление", "heating_type" => "Отопление", "sewerage_type" => "Канализация", "water_supply_type" => "Водоснабжение", "direction_departure" => "Направление выезда", "has_water_supply" => "Водоснабжение", "has_garage" => "Гараж", "has_bathhouse" => "Баня", "has_plantings" => "Посадки", "has_toilet" => "Туалет", "has_greenhouse" => "Теплица", "has_asphalt_road" => "Асфальтовая дорога", "has_gas_supply" => "Газ", "has_security" => "Охрана", "is_available_in_winter" => "Доступ зимой", "is_shop_near" => "Магазин рядом", "is_water_near" => "Водоем рядом", "is_forest_near" => "Лес рядом", "distance_to_bus_stop_type" => "Расстояние до автобусной остановки", "summer_cottage_area" => "", "railway_stations" => "", "roads" => "", "has_sewerage" => "Канализация", "floor_type" => "Этаж", "ceiling_height" => "Высота потолков", "electrical_power" => "Мощность электросети", "parking_capacity" => "Парковка, количество мест", "commercial_object" => "Тип здания", "commercial_building_type" => "Тип здания", "own_entrance" => "Отдельный вход", "purposes" => "Назначение", "has_furniture" => "Мебель", "has_refrigerator" => "Холодильник", "rent_period" => "Срок аренды", "is_housing_payments_included" => "Коммунальные включены", "utility_payments" => "Коммунальные платежи", ]; //$r = false; $data = (array) $json->__INITIAL_STATE__->OffersSearch->offers->result; //$this->__log('found - '.count($data)); //foreach ($data as $_) { // $_ = $_->result; // $r = is_array($_) && count($_) && isset(current($_)->_id); // if ($r) { // $data = $_; // break; // } //} //$this->__log('found - '.count($data)); //$this->__log('isOk - '.$r); //if (!$r) return []; $items = []; foreach ($data as $_) { if (!$_->photos_count) $_->photos = []; foreach ($_->photos as &$item) { $item = "https://cdn.n1.ru" . $item->original; } unset($item); if (!$_->contact_phones) $_->contact_phones = []; foreach ($_->contact_phones as &$item) { $item = $item->value; } unset($item); $item = (object) [ id => $_->_id, date => date("Y-m-d H:i:s", strtotime(isset($_->modification_date) ? $_->modification_date : $_->creation_date)), url => "https:" . $_->url, title => $titles[$_->_id], images => $_->photos, is_agency => $_->agency_code || $_->is_agency, seller => ($_->contact_organization ? '"' . $_->contact_organization . '", ' : "") . $_->contact_name, phone => $_->contact_phones, params => "", ]; $params = new stdClass(); foreach ($__params as $k => $v) { if (empty($_->params->$k)) continue; if (!$v) $v = $k; if (is_array($_->params->$k)) { $_->params->$k = $_->params->$k[0]; } if (is_object($_->params->$k)) { foreach (["name_ru", "title", "name"] as $p) { if (property_exists($_->params->$k, $p)) { $_->params->$k = $_->params->$k->$p; break; } } } if (!empty($_->params->$k) && is_scalar($_->params->$k)) { $params->$k = preg_match('/\_area$/i', $k) ? 0.01 * $_->params->$k : $_->params->$k; $item->params .= "{$v}: {$params->$k}; "; } } $item->params = trim($item->params); $item->description = strip_tags($_->params->description); $item->price = $_->params->price; $item->rooms = $params->rooms_count; $item->level = $params->floor; $item->levels = $params->floors_count; $item->raion = $params->microdistrict; $item->s = __floatval($params->total_area); $item->land = __floatval($params->homestead_area); $item->house_number = $params->house_number; $item->material = $params->house_material_type; $item->address = $params->reference_point; if (is_array($_->params->house_addresses) && count($_->params->house_addresses)) { $_->params->house_addresses = $_->params->house_addresses[0]; $item->address = $_->params->house_addresses->street->name_ru . " " . $_->params->house_addresses->street->abbr_raw_ru; if ($_->params->house_addresses->house_number) $item->address .= ", " . $_->params->house_addresses->house_number; $item->address = trim($item->address); } else { $item->address = $params->street; if ($item->house_number) $item->address .= ", " . $item->house_number; } $items[$item->id] = $item; } //$this->__log('parsed - '.count($items)); // //return []; return $items; } public function getItem($item, $n = 0) { if($n){ $err = "getItem({$item->id}) - error receiving data"; $this->update_proxy(1, -1, $err); if($n > $this->fails) return $this->__log($err); } $this->__log("getItem: {$item->url}"); if (!$item->phone) { $this->__log("skipped..."); return false; } $item->display = !$item->is_agency && $this->checkPhone($item->phone); $item->phone = implode("\n", $item->phone); $item->images = implode("\n", $item->images); isset($this->olditems[$item->id]) ? $this->updateItem($item) : $this->saveItem($item); return $item; } public function create_url($urlsfx, $p = 1) { $url = "https://" . $this->site->host . "/{$this->deals[$this->deal_type]}/{$urlsfx}/?creation_date_min=30"; if ($p > 1) $url .= "&page={$p}"; $url .= "&limit=100&sort=-date"; if ($this->category != "kommercheskaya_nedvizhimost") $url .= "&author=owner"; return $url; } public function isValidItemHtml($html, $item) { return 0; } public function session_start() { $this->cookies = "{$_ENV[ROOT]}/../logs/{$this->site->code}.cookie"; @unlink($this->cookies); } public function session_end() { } public function _session_end() { @unlink("{$_ENV[ROOT]}/../pids/{$this->site->code}.pid"); @unlink($this->cookies); } public function grab() { $sections = [ [deal_type => "prodam", category => "kvartiry"], [deal_type => "sdam", category => "kvartiry"], [deal_type => "prodam", category => "kommercheskaya_nedvizhimost"], [deal_type => "sdam", category => "kommercheskaya_nedvizhimost"], [deal_type => "prodam", category => "doma_dachi_kottedzhi"], [deal_type => "sdam", category => "doma_dachi_kottedzhi"], [deal_type => "prodam", category => "zemelnye_uchastki"], [deal_type => "sdam", category => "zemelnye_uchastki"], ]; $city = conf::$city->id; foreach ($sections as $section) { $deal_type = $section["deal_type"]; $category = $section["category"]; if(!isset($this->sections[$deal_type][$category])) continue; $this->__log("\n\n=========\n grab($city, $deal_type, $category);\n=========\n"); parent::grab($city, $deal_type, $category); } $this->_session_end(); } public function cron() { //return parent::__cron(); $logfile = "{$_ENV[ROOT]}/../logs/{$this->site->code}.log"; $pidfile = "{$_ENV[ROOT]}/../pids/{$this->site->code}.pid"; if (is_file($pidfile)) { $pid = file_get_contents($pidfile); shell_exec("kill -9 {$pid} > /dev/null 2>&1"); } $cmd = "php {$_ENV[ROOT]}/grabber.php {$this->site->code} > {$logfile} 2>&1 & echo $!"; // echo "[" . date("Y-M-d H:i:s") . "] " . $cmd . "\n"; $pid = shell_exec($cmd); file_put_contents($pidfile, $pid); } public function get_proxy($proxy = "", $reset = 0) { $proxy = $this->get_proxylist(); $this->__proxy = (object) [ip => "", port => "", login => "", pass => ""]; // $this->proxy = "10.0.0.1:8800"; $this->proxy = $proxy[mt_rand(0, count($proxy) - 1)]; list($this->__proxy->ip, $this->__proxy->port, $this->__proxy->login, $this->__proxy->pass) = explode(":", $this->proxy); $this->proxy = $this->__proxy->ip . ":" . $this->__proxy->port; return $this->proxy; } public function update_proxy($n, $status = "", $error = "") { return false; } }