summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore12
-rw-r--r--app/__init__.py91
-rw-r--r--app/acmespb.py133
-rw-r--r--app/static/app.js191
-rw-r--r--app/static/autocomplete.js135
-rw-r--r--app/static/style.css17
-rw-r--r--app/templates/base.html39
-rw-r--r--app/templates/index.html2
-rw-r--r--app/test.py15
-rw-r--r--requirements.txt4
-rw-r--r--server.py7
11 files changed, 646 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..685b2cd
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,12 @@
+*.pyc
+__pycache__/
+
+instance/
+
+.pytest_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+*.egg-info/ \ No newline at end of file
diff --git a/app/__init__.py b/app/__init__.py
new file mode 100644
index 0000000..8ee21b9
--- /dev/null
+++ b/app/__init__.py
@@ -0,0 +1,91 @@
+import os
+import time
+
+from . import acmespb
+from flask import Flask, render_template
+from flask_socketio import SocketIO, emit
+
+socketio = SocketIO()
+
+
+def create_app(test_config=None):
+ app = Flask(__name__, instance_relative_config=True)
+ app.config.from_mapping(
+ SECRET_KEY='dev',
+ DATABASE=os.path.join(app.instance_path, 'app.sqlite'),
+ )
+
+ if test_config is None:
+ # load the instance config, if it exists, when not testing
+ app.config.from_pyfile('config.py', silent=True)
+ else:
+ # load the test config if passed in
+ app.config.from_mapping(test_config)
+
+ # ensure the instance folder exists
+ try:
+ os.makedirs(app.instance_path)
+ except OSError:
+ pass
+
+ socketio.init_app(app)
+
+ @app.route('/')
+ def hello():
+ return render_template('index.html')
+
+ @socketio.on('get_hints')
+ def handle_get_hints_event(q):
+ print('[get_hints] id=%d, query=%s' % (q['id'], q['query']))
+ if len(q['query']) < 3:
+ response = {
+ 'id': q['id'],
+ 'error': "query is too short"
+ }
+ emit('hints', response)
+ return
+ results = acmespb.search(q['query'])
+ response = {
+ 'id': q['id'],
+ 'response': results
+ }
+ emit('hints', response)
+
+ @socketio.on('get_offers')
+ def handle_get_offers_event(q):
+ print('[get_offers] id=%d, query=%s' % (q['id'], q['query']))
+ target_url, trade_names = acmespb.trade_names(q['query'])
+ if trade_names:
+ response = {
+ 'id': q['id'],
+ "response": trade_names
+ }
+ emit('hints', response)
+ return
+
+ page = 1
+ pages = 0
+ target_url = None
+ while pages == 0 or page <= pages:
+ target_url, pages, offers = acmespb.offers(q['query'], page=page, target_url=target_url)
+ print("[%d] pages=%d, target_url=%s" % (page, pages, target_url))
+ response = {
+ 'id': q['id'],
+ 'offers': [offer.as_dict() for offer in offers],
+ 'page': page,
+ 'pages': pages
+ }
+ emit('offers', response)
+
+ time.sleep(0.5)
+ page += 1
+
+ response = {
+ 'id': q['id'],
+ 'end': True
+ }
+ emit('offers', response)
+
+ # TODO empty response
+
+ return app
diff --git a/app/acmespb.py b/app/acmespb.py
new file mode 100644
index 0000000..5ebc41f
--- /dev/null
+++ b/app/acmespb.py
@@ -0,0 +1,133 @@
+import requests
+import urllib.parse
+import json
+import re
+import math
+import hashlib
+
+from bs4 import BeautifulSoup
+
+headers = {
+ 'Referer': 'https://www.acmespb.ru/',
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.89 Safari/537.36'
+}
+proxies = {
+ 'http': 'socks5://127.0.0.1:1079',
+ 'https': 'socks5://127.0.0.1:1079'
+}
+per_page = 50
+
+session = requests.Session()
+session.proxies.update(proxies)
+session.headers.update(headers)
+
+
+class AcmeException(Exception):
+ pass
+
+
+class AcmePharmacy:
+ def __init__(self, name='', address='', phone='', geo=None):
+ self.name = name
+ self.address = address
+ self.phone = phone
+ self.geo = geo
+
+ def as_dict(self):
+ dict = self.__dict__
+ dict['hash'] = hashlib.md5(("%s|%s" % (self.address, self.name)).encode('utf-8')).hexdigest()
+ return dict
+
+
+class AcmeOffer:
+ def __init__(self, name='', country='', pharmacy=None, price=None):
+ self.name = name
+ self.country = country
+ self.pharmacy = pharmacy
+ self.price = price
+
+ def as_dict(self):
+ dict = self.__dict__
+ dict['pharmacy'] = self.pharmacy.as_dict()
+ return dict
+
+
+def search(query):
+ url = "https://www.acmespb.ru/lib/autocomplete.php?term=" + urllib.parse.quote(query)
+ r = session.get(url, allow_redirects=False)
+ if r.text == "":
+ return []
+
+ r.encoding = "utf-8"
+ return json.loads(r.text)
+
+
+def trade_names(query):
+ url = "https://www.acmespb.ru/search.php"
+ r = session.post(url, {"free_str": query}, allow_redirects=False)
+ if r.status_code != 301:
+ raise AcmeException("status_code is %d" % (r.status_code,))
+ if '/trade/' not in r.headers["location"]:
+ return r.headers["location"], None
+
+ r = session.get(r.headers["location"], allow_redirects=False)
+ r.encoding = "utf-8"
+ soup = BeautifulSoup(r.text, "html.parser")
+ trades = soup.find(id="trades")
+ return None, [opt.string for opt in trades.find_all("option") if opt["value"] != "all"]
+
+
+def _get_location(query):
+ url = "https://www.acmespb.ru/search.php"
+ data = {"free_str": query}
+ r = session.post(url, data, allow_redirects=False)
+ return r.headers["location"]
+
+
+def offers(query, target_url=None, page=1):
+ if target_url is None:
+ target_url = _get_location(query)
+
+ data = {
+ "free_str": query,
+ "page": page
+ }
+ r = session.post(target_url, data, allow_redirects=False)
+ r.encoding = "utf-8"
+ if r.status_code != 200:
+ raise AcmeException("status_code is %d, expected 200" % (r.status_code,))
+
+ pages = 1
+
+ soup = BeautifulSoup(r.text, "html.parser")
+ p = soup.find("p", class_="red")
+ if p:
+ total_matches = int(re.findall("([0-9]+)", p.string)[0])
+ pages = math.ceil(total_matches / per_page)
+
+ offers = []
+ for trow in soup.find_all('div', class_='trow'):
+ if 'thead' in trow['class']:
+ continue
+
+ name = trow.select_one('.cell.name p.sra').text
+ country = trow.select_one('.cell.country').text
+ phname = trow.select_one('.cell.pharm a').text
+ price = float(trow.select_one('.cell.pricefull').text)
+
+ # parse address, geo coordinates and phone number
+ addr_div = trow.select_one('.cell.address')
+ phone = re.findall('тел\.([^<]+)', addr_div.text)[0].strip()
+
+ addr_link = addr_div.select_one('a')
+ address = addr_link.text
+
+ geo = re.findall('text=([0-9\.]+),([0-9\.]+)', addr_link['href'])[0]
+ geo = list(map(lambda x: float(x), geo))
+
+ acmepharm = AcmePharmacy(name=phname, address=address, phone=phone, geo=geo)
+ acmeoffer = AcmeOffer(name=name, country=country, price=price, pharmacy=acmepharm)
+
+ offers.append(acmeoffer)
+
+ return target_url, pages, offers \ No newline at end of file
diff --git a/app/static/app.js b/app/static/app.js
new file mode 100644
index 0000000..bc4b89e
--- /dev/null
+++ b/app/static/app.js
@@ -0,0 +1,191 @@
+class Search {
+ constructor() {
+ this.searchDebounced = _.debounce((query) => {
+ if (query.length < 3)
+ return;
+ this.socket.emit('get_hints', {
+ id: this.updateRequestId(),
+ query
+ });
+ }, 150);
+
+ let field = document.getElementById('queryInput');
+ let btn = document.getElementById('querySubmit');
+
+ this.autoComplete = new Autocomplete(field, {
+ data: [],
+ maximumItems: 10,
+ onInput: (value) => {
+ this.searchDebounced(value);
+ },
+ onSelectItem: ({label}) => {
+ // console.log('selected:', label)
+ },
+ highlightClass: 'text-danger'
+ });
+
+ btn.addEventListener('click', this.onSubmit);
+ field.addEventListener('keydown', this.onInputKeyDown);
+
+ this.btn = btn;
+ this.field = field;
+
+ this.socket = io();
+ this.socket.on('hints', this.onHints);
+ this.socket.on('offers', this.onOffers)
+ }
+
+ updateRequestId() {
+ this.requestId = requestId();
+ return this.requestId;
+ }
+
+ onInputKeyDown = (e) => {
+ if (e.keyCode === 10 || e.keyCode === 13)
+ this.onSubmit();
+ }
+
+ onSubmit = (e) => {
+ if (this.isLocked())
+ return;
+
+ this.lockButton('Загрузка...');
+
+ gMaps.removeAllPoints();
+ this.socket.emit('get_offers', {
+ id: this.updateRequestId(),
+ query: this.field.value
+ });
+ }
+
+ onHints = (data) => {
+ if (data.id !== this.requestId)
+ return;
+
+ this.unlockButton();
+
+ if (data.error) {
+ console.warn(data.error);
+ return;
+ }
+
+ this.autoComplete.setData(data.response.map(item => {
+ return {label: item, value: ''};
+ }));
+ this.autoComplete.renderIfNeeded();
+ }
+
+ onOffers = (data) => {
+ if (data.id !== this.requestId)
+ return;
+
+ if (data.end) {
+ this.unlockButton();
+ return;
+ } else {
+ this.lockButton(data.pages > 1 ? `${data.page} из ${data.pages}` : null);
+ }
+
+ for (let offer of data.offers)
+ gMaps.addOffer(offer);
+ }
+
+ isLocked() {
+ return this.btn.classList.contains('disabled');
+ }
+
+ lockButton(text) {
+ if (text !== null)
+ this.btn.innerText = text;
+ this.btn.classList.add('disabled');
+ }
+
+ unlockButton() {
+ this.btn.classList.remove('disabled');
+ this.btn.innerText = 'Поиск';
+ }
+}
+
+
+class Maps {
+ constructor() {
+ /**
+ * @type {ymaps.Map}
+ */
+ this.map = null;
+ ymaps.ready(this.onInit);
+
+ this.places = {};
+ }
+
+ onInit = () => {
+ this.map = new ymaps.Map("mapContainer", {
+ center: [59.94, 30.32],
+ zoom: 11
+ });
+ this.map.controls.remove('searchControl');
+ }
+
+ addPoint({geo, offersRef, hint, pharmacyName, pharmacyAddress, pharmacyPhone}) {
+ let mark = new ymaps.Placemark(geo, {
+ hintContent: hint,
+ }, {
+ preset: 'islands#dotIcon',
+ openEmptyBalloon: true,
+ iconColor: '#3caa3c'
+ });
+ mark.events.add('balloonopen', e => {
+ let lines = offersRef.map(offer => {
+ return `${offer.name} (${offer.price} руб.)`
+ });
+ let html = `<b>${pharmacyName}</b><br>`;
+ html += `${pharmacyAddress}<br>`;
+ html += `тел: ${pharmacyPhone}<br><br>`;
+ html += lines.join('\n');
+ mark.properties.set('balloonContent', html);
+ });
+ this.map.geoObjects.add(mark);
+ return mark;
+ }
+
+ removeAllPoints() {
+ this.map.geoObjects.removeAll();
+ }
+
+ addOffer(offer) {
+ // console.log('[addOffer]', offer);
+ let hash = offer.pharmacy.hash;
+ if (hash in this.places)
+ this.places[hash].offers.push(offer);
+ else {
+ this.places[hash] = {
+ offers: [offer],
+ };
+ this.places[hash].mark = this.addPoint({
+ geo: offer.pharmacy.geo,
+ hint: offer.pharmacy.name,
+ pharmacyName: offer.pharmacy.name,
+ pharmacyAddress: offer.pharmacy.address,
+ pharmacyPhone: offer.pharmacy.phone,
+ offersRef: this.places[hash].offers
+ });
+ }
+ }
+}
+
+
+function requestId() {
+ return _.random(1, 99999999);
+}
+
+
+let gMaps, gSearch;
+
+window.addEventListener('DOMContentLoaded', function() {
+ gSearch = new Search();
+ gMaps = new Maps();
+
+ // document.getElementById('test').addEventListener('click', () => {
+ // gMaps.addTestPoint();
+ // });
+}); \ No newline at end of file
diff --git a/app/static/autocomplete.js b/app/static/autocomplete.js
new file mode 100644
index 0000000..87b42a5
--- /dev/null
+++ b/app/static/autocomplete.js
@@ -0,0 +1,135 @@
+const DEFAULTS = {
+ treshold: 2,
+ maximumItems: 5,
+ highlightTyped: true,
+ highlightClass: 'text-primary',
+};
+
+class Autocomplete {
+ constructor(field, options) {
+ this.field = field;
+ this.options = Object.assign({}, DEFAULTS, options);
+ this.dropdown = null;
+
+ field.parentNode.classList.add('dropdown');
+ field.setAttribute('data-toggle', 'dropdown');
+ field.classList.add('dropdown-toggle');
+
+ const dropdown = ce(`<div class="dropdown-menu" ></div>`);
+ if (this.options.dropdownClass)
+ dropdown.classList.add(this.options.dropdownClass);
+
+ insertAfter(dropdown, field);
+
+ this.dropdown = new bootstrap.Dropdown(field, this.options.dropdownOptions)
+
+ field.addEventListener('click', (e) => {
+ if (this.createItems() === 0) {
+ // prevent show empty
+ e.stopPropagation();
+ this.dropdown.hide();
+ // field.dropdown('hide');
+ }
+ });
+
+ field.addEventListener('input', () => {
+ if (this.options.onInput)
+ this.options.onInput(this.field.value);
+ this.renderIfNeeded();
+ });
+
+ field.addEventListener('keydown', (e) => {
+ if (e.keyCode === 27) {
+ this.dropdown.hide();
+ return;
+ }
+ });
+ }
+
+ setData(data) {
+ this.options.data = data;
+ }
+
+ renderIfNeeded() {
+ if (this.createItems() > 0) {
+ this.dropdown.show();
+ // field.dropdown('show');
+ } else {
+ // sets up positioning
+ this.field.click();
+ }
+ }
+
+ createItem(lookup, item) {
+ let label;
+ if (this.options.highlightTyped) {
+ const idx = item.label.toLowerCase().indexOf(lookup.toLowerCase());
+ const className = Array.isArray(this.options.highlightClass) ? this.options.highlightClass.join(' ')
+ : (typeof this.options.highlightClass == 'string' ? this.options.highlightClass : '')
+ label = item.label.substring(0, idx)
+ + `<span class="${className}">${item.label.substring(idx, idx + lookup.length)}</span>`
+ + item.label.substring(idx + lookup.length, item.label.length);
+ } else {
+ label = item.label;
+ }
+ return ce(`<button type="button" class="dropdown-item" data-value="${item.value}">${label}</button>`);
+ }
+
+ createItems() {
+ const lookup = this.field.value;
+ if (lookup.length < this.options.treshold) {
+ this.dropdown.hide();
+ // field.dropdown('hide');
+ return 0;
+ }
+
+ const items = this.field.nextSibling;
+ items.innerHTML = '';
+
+ let count = 0;
+ for (let i = 0; i < this.options.data.length; i++) {
+ const {label, value} = this.options.data[i];
+ const item = {label, value};
+ if (item.label.toLowerCase().indexOf(lookup.toLowerCase()) >= 0) {
+ items.appendChild(this.createItem(lookup, item));
+ if (this.options.maximumItems > 0 && ++count >= this.options.maximumItems)
+ break;
+ }
+ }
+
+ this.field.nextSibling.querySelectorAll('.dropdown-item').forEach((item) => {
+ item.addEventListener('click', (e) => {
+ let dataValue = e.target.getAttribute('data-value');
+ this.field.value = e.target.innerText;
+ if (this.options.onSelectItem) {
+ this.options.onSelectItem({
+ value: e.target.value,
+ label: e.target.innerText,
+ });
+ }
+ this.dropdown.hide();
+ })
+ });
+
+ return items.childNodes.length;
+ }
+}
+
+/**
+ * @param html
+ * @returns {Node}
+ */
+function ce(html) {
+ let div = document.createElement('div');
+ div.innerHTML = html;
+ return div.firstChild;
+}
+
+/**
+ * @param elem
+ * @param refElem
+ * @returns {*}
+ */
+function insertAfter(elem, refElem) {
+ return refElem.parentNode.insertBefore(elem, refElem.nextSibling)
+} \ No newline at end of file
diff --git a/app/static/style.css b/app/static/style.css
new file mode 100644
index 0000000..303ffdc
--- /dev/null
+++ b/app/static/style.css
@@ -0,0 +1,17 @@
+/*.acme-container {*/
+/* margin-top: 1.5rem;*/
+/*}*/
+
+#test {
+ position: absolute;
+ color: #fff;
+ background-color: red;
+ opacity: 0.25;
+ top: 0;
+ right: 0;
+ padding: 5px 8px;
+ cursor: pointer;
+}
+#test:hover {
+ opacity: 1;
+} \ No newline at end of file
diff --git a/app/templates/base.html b/app/templates/base.html
new file mode 100644
index 0000000..aeb42c6
--- /dev/null
+++ b/app/templates/base.html
@@ -0,0 +1,39 @@
+<!doctype html>
+<html lang="ru" class="h-100 mh-100">
+<head>
+ <meta charset="utf-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-giJF6kkoqNQ00vy+HMDP7azOuL0xtbfIcaT9wjKHr8RbDVddVHyTfAAsrekwKmP1" crossorigin="anonymous">
+ <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+
+ <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0-beta1/dist/js/bootstrap.bundle.min.js" integrity="sha384-ygbV9kiqUc6oa4msXn9868pTtWMgiQaeYH7/t7LECLbyPA2x65Kgf80OJFdroafW" crossorigin="anonymous"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/3.0.4/socket.io.js" integrity="sha512-aMGMvNYu8Ue4G+fHa359jcPb1u+ytAF+P2SCb+PxrjCdO3n3ZTxJ30zuH39rimUggmTwmh2u7wvQsDTHESnmfQ==" crossorigin="anonymous"></script>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.20/lodash.min.js" integrity="sha512-90vH1Z83AJY9DmlWa8WkjkV79yfS2n2Oxhsi2dZbIv0nC4E6m5AbH8Nh156kkM7JePmqD6tcZsfad1ueoaovww==" crossorigin="anonymous"></script>
+
+ <script src="https://api-maps.yandex.ru/2.1/?apikey=ce936229-3ef4-41b1-96c0-270bcf8ff341&lang=ru_RU" type="text/javascript"></script>
+
+ <script src="{{ url_for('static', filename='autocomplete.js') }}"></script>
+ <script src="{{ url_for('static', filename='app.js') }}"></script>
+
+ <title>{% block title %}{% endblock %}</title>
+</head>
+<body class="h-100 mh-100">
+ <div class="container h-100 pt-4 pb-4">
+ <div class="h-100 d-flex flex-column bd-highlight">
+ <div>
+ <form class="mb-4" onsubmit="return false">
+ <div class="input-group">
+ <input type="text" class="form-control" id="queryInput" placeholder="Введите название препарата" autocomplete="off">
+ <button type="submit" class="btn btn-outline-primary" id="querySubmit">Поиск</button>
+ </div>
+ </form>
+ </div>
+ <div class="flex-grow-1" id="mapContainer">
+ <!-- maps -->
+ </div>
+ </div>
+ </div>
+ <div id="test">test</div>
+</body>
+</html> \ No newline at end of file
diff --git a/app/templates/index.html b/app/templates/index.html
new file mode 100644
index 0000000..5cb6467
--- /dev/null
+++ b/app/templates/index.html
@@ -0,0 +1,2 @@
+{% extends "base.html" %}
+
diff --git a/app/test.py b/app/test.py
new file mode 100644
index 0000000..a0481e2
--- /dev/null
+++ b/app/test.py
@@ -0,0 +1,15 @@
+import acmespb
+import sys
+from pprint import pprint
+
+if __name__ == "__main__":
+ #pprint(acmespb.trade_names("Марена красильная корневища и корни"))
+ page = 1
+ pages = 0
+ target_url = None
+ while pages == 0 or page <= pages:
+ target_url, pages, offers = acmespb.offers("Верошпирон", page=page, target_url=target_url)
+ print("[%d] pages=%d, target_url=%s" % (page, pages, target_url))
+ for offer in offers:
+ print(offer.as_dict())
+ page += 1 \ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..41d652e
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,4 @@
+requests~=2.25.1
+requests[socks]
+beautifulsoup4~=4.9.3
+Flask~=1.1.2 \ No newline at end of file
diff --git a/server.py b/server.py
new file mode 100644
index 0000000..a243faa
--- /dev/null
+++ b/server.py
@@ -0,0 +1,7 @@
+#!/bin/env python
+from app import create_app, socketio
+
+app = create_app()
+
+if __name__ == '__main__':
+ socketio.run(app)
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 09:37:58 +0000