From 38474234431fcf1dae50c00cdf649b4102c0faf9 Mon Sep 17 00:00:00 2001 From: Evgeny Zinoviev Date: Sat, 18 Feb 2023 20:07:00 +0300 Subject: wip --- .gitignore | 5 ++ check-ratio.py | 4 +- database.py | 152 ------------------------------------------------- database/__init__.py | 1 + database/database.py | 152 +++++++++++++++++++++++++++++++++++++++++++++++++ grab-retronews.py | 132 ++++++++++++++++++++++++++++++------------ retronews.py | 48 ---------------- retronews/__init__.py | 8 +++ retronews/retronews.py | 50 ++++++++++++++++ 9 files changed, 313 insertions(+), 239 deletions(-) create mode 100644 .gitignore delete mode 100644 database.py create mode 100644 database/__init__.py create mode 100644 database/database.py delete mode 100644 retronews.py create mode 100644 retronews/__init__.py create mode 100644 retronews/retronews.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49938ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/.idea +/venv +/*.sqlite3 +__pycache__ +/test.py \ No newline at end of file diff --git a/check-ratio.py b/check-ratio.py index bd546bf..6c85283 100755 --- a/check-ratio.py +++ b/check-ratio.py @@ -20,5 +20,5 @@ if __name__ == '__main__': print(f'ERROR: {doc["collection_id"]}/{doc["page_id"]}/{page}: width or height is zero') continue ratio = width/height - # TODO - # print(f'[]') \ No newline at end of file + if ratio >= 0.8: + print(f'{doc["collection_id"]}/{doc["doc_id"]}/{page}: {ratio}') diff --git a/database.py b/database.py deleted file mode 100644 index b67f4d0..0000000 --- a/database.py +++ /dev/null @@ -1,152 +0,0 @@ -import sqlite3 -import logging -import os.path -import retronews -import threading - -from typing import Optional - - -class Database: - SCHEMA = 6 - - def __init__(self): - self.logger = logging.getLogger(self.__class__.__name__) - - file = os.path.join(os.path.dirname(__file__), 'mdf-retrobase.sqlite3') - self.sqlite = sqlite3.connect(file, check_same_thread=False) - self.lock = threading.Lock() - - sqlite_version = self._get_sqlite_version() - self.logger.debug(f'SQLite version: {sqlite_version}') - - schema_version = self.schema_get_version() - self.logger.debug(f'Schema version: {schema_version}') - - self.schema_init(schema_version) - self.schema_set_version(self.SCHEMA) - - def __del__(self): - if self.sqlite: - self.sqlite.commit() - self.sqlite.close() - - def _get_sqlite_version(self) -> str: - cursor = self.sqlite.cursor() - cursor.execute("SELECT sqlite_version()") - return cursor.fetchone()[0] - - def schema_get_version(self) -> int: - cursor = self.sqlite.execute('PRAGMA user_version') - return int(cursor.fetchone()[0]) - - def schema_set_version(self, v) -> None: - self.sqlite.execute('PRAGMA user_version={:d}'.format(v)) - self.logger.info(f'Schema set to {v}') - - def cursor(self) -> sqlite3.Cursor: - return self.sqlite.cursor() - - def commit(self) -> None: - return self.sqlite.commit() - - def schema_init(self, version: int) -> None: - cursor = self.cursor() - - if version < 1: - # timestamps - cursor.execute("""CREATE TABLE IF NOT EXISTS mdf_links ( - issue_date TEXT PRIMARY KEY, - url TEXT NOT NULL, - pages INTEGER NOT NULL - )""") - - if version < 2: - cursor.execute("""CREATE TABLE IF NOT EXISTS mdf_pages ( - collection_id INTEGER NOT NULL, - doc_id INTEGER NOT NULL, - page INTEGER NOT NULL, - height INTEGER NOT NULL, - width INTEGER NOT NULL, - dpi INTEGER NOT NULL - )""") - cursor.execute("""CREATE UNIQUE INDEX mdf_pages_idx ON mdf_pages (collection_id, doc_id, page)""") - - if version < 3: - cursor.execute("ALTER TABLE mdf_pages ADD fail INTEGER NOT NULL") - - if version < 4: - cursor.execute("""CREATE INDEX mdf_pages_fail_idx ON mdf_pages (fail)""") - - if version < 5: - for col in ('collection_id', 'doc_id'): - cursor.execute(f"ALTER TABLE mdf_links ADD {col} INTEGER NOT NULL DEFAULT '0'") - cursor.execute("CREATE INDEX mdf_links_col_doc_idx ON mdf_links (collection_id, doc_id)") - - if version < 6: - cursor.execute("DROP INDEX mdf_links_col_doc_idx") - cursor.execute("CREATE UNIQUE INDEX mdf_links_col_doc_idx ON mdf_links (collection_id, doc_id)") - - self.commit() - - def add_link(self, issue_date: str, url: str, pages: int): - with self.lock: - self.cursor().execute("REPLACE INTO mdf_links (issue_date, url, pages) VALUES (?, ?, ?)", - (issue_date, url, str(pages))) - self.commit() - - def add_page(self, collection_id: int, doc_id: int, page: int, width: int, height: int, dpi: int): - with self.lock: - self.cursor().execute("INSERT INTO mdf_pages (collection_id, doc_id, page, width, height, dpi, fail) VALUES (?, ?, ?, ?, ?, ?, 0)", - (collection_id, doc_id, page, width, height, dpi)) - self.commit() - - def update_page(self, collection_id: int, doc_id: int, page: int, width: int, height: int, dpi: int): - with self.lock: - self.cursor().execute("UPDATE mdf_pages SET width=?, height=?, dpi=?, fail=0 WHERE collection_id=? AND doc_id=? AND page=?", - (width, height, dpi, collection_id, doc_id, page)) - self.commit() - - def add_page_failed(self, collection_id, doc_id, page): - with self.lock: - self.cursor().execute("INSERT INTO mdf_pages (collection_id, doc_id, page, width, height, dpi, fail) VALUES (?, ?, ?, 0, 0, 0, 1)", - (collection_id, doc_id, page)) - self.commit() - - def get_existing_pages(self, fail=0): - cur = self.cursor() - cur.execute("SELECT collection_id, doc_id, page FROM mdf_pages WHERE fail=?", (fail,)) - return cur.fetchall() - - def get_documents(self, range: Optional[tuple[str, str]] = None): - cur = self.cursor() - docs = [] - - sql = "SELECT issue_date, url, pages FROM mdf_links" - if range: - sql += f" WHERE issue_date BETWEEN '{range[0]}' AND '{range[1]}'" - cur.execute(sql) - for issue_date, url, pages in cur.fetchall(): - pub_date, collection_id, doc_id = retronews.parse_url(url) - docs.append(dict( - collection_id=collection_id, - doc_id=doc_id, - pages=pages - )) - - return docs - - def get_doc_pages(self, collection_id, doc_id): - cur = self.cursor() - cur.execute("SELECT page, width, height, dpi FROM mdf_pages WHERE collection_id=? AND doc_id=?", - (collection_id, doc_id)) - return cur.fetchall() - - def fix_documents(self): - cur = self.cursor() - cur.execute("SELECT issue_date, url FROM mdf_links") - for issue_date, url in cur.fetchall(): - pub_date, cid, did = retronews.parse_url(url) - cur.execute("UPDATE mdf_links SET collection_id=?, doc_id=? WHERE issue_date=?", - (cid, did, issue_date)) - self.commit() diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..ef3f969 --- /dev/null +++ b/database/__init__.py @@ -0,0 +1 @@ +from .database import Database diff --git a/database/database.py b/database/database.py new file mode 100644 index 0000000..04902f1 --- /dev/null +++ b/database/database.py @@ -0,0 +1,152 @@ +import sqlite3 +import logging +import os.path +import retronews +import threading + +from typing import Optional + + +class Database: + SCHEMA = 6 + + def __init__(self): + self.logger = logging.getLogger(self.__class__.__name__) + + file = os.path.join(os.path.dirname(__file__), '..', 'mdf-retrobase.sqlite3') + self.sqlite = sqlite3.connect(file, check_same_thread=False) + self.lock = threading.Lock() + + sqlite_version = self._get_sqlite_version() + self.logger.debug(f'SQLite version: {sqlite_version}') + + schema_version = self.schema_get_version() + self.logger.debug(f'Schema version: {schema_version}') + + self.schema_init(schema_version) + self.schema_set_version(self.SCHEMA) + + def __del__(self): + if self.sqlite: + self.sqlite.commit() + self.sqlite.close() + + def _get_sqlite_version(self) -> str: + cursor = self.sqlite.cursor() + cursor.execute("SELECT sqlite_version()") + return cursor.fetchone()[0] + + def schema_get_version(self) -> int: + cursor = self.sqlite.execute('PRAGMA user_version') + return int(cursor.fetchone()[0]) + + def schema_set_version(self, v) -> None: + self.sqlite.execute('PRAGMA user_version={:d}'.format(v)) + self.logger.info(f'Schema set to {v}') + + def cursor(self) -> sqlite3.Cursor: + return self.sqlite.cursor() + + def commit(self) -> None: + return self.sqlite.commit() + + def schema_init(self, version: int) -> None: + cursor = self.cursor() + + if version < 1: + # timestamps + cursor.execute("""CREATE TABLE IF NOT EXISTS mdf_links ( + issue_date TEXT PRIMARY KEY, + url TEXT NOT NULL, + pages INTEGER NOT NULL + )""") + + if version < 2: + cursor.execute("""CREATE TABLE IF NOT EXISTS mdf_pages ( + collection_id INTEGER NOT NULL, + doc_id INTEGER NOT NULL, + page INTEGER NOT NULL, + height INTEGER NOT NULL, + width INTEGER NOT NULL, + dpi INTEGER NOT NULL + )""") + cursor.execute("""CREATE UNIQUE INDEX mdf_pages_idx ON mdf_pages (collection_id, doc_id, page)""") + + if version < 3: + cursor.execute("ALTER TABLE mdf_pages ADD fail INTEGER NOT NULL") + + if version < 4: + cursor.execute("""CREATE INDEX mdf_pages_fail_idx ON mdf_pages (fail)""") + + if version < 5: + for col in ('collection_id', 'doc_id'): + cursor.execute(f"ALTER TABLE mdf_links ADD {col} INTEGER NOT NULL DEFAULT '0'") + cursor.execute("CREATE INDEX mdf_links_col_doc_idx ON mdf_links (collection_id, doc_id)") + + if version < 6: + cursor.execute("DROP INDEX mdf_links_col_doc_idx") + cursor.execute("CREATE UNIQUE INDEX mdf_links_col_doc_idx ON mdf_links (collection_id, doc_id)") + + self.commit() + + def add_link(self, issue_date: str, url: str, pages: int): + with self.lock: + self.cursor().execute("REPLACE INTO mdf_links (issue_date, url, pages) VALUES (?, ?, ?)", + (issue_date, url, str(pages))) + self.commit() + + def add_page(self, collection_id: int, doc_id: int, page: int, width: int, height: int, dpi: int): + with self.lock: + self.cursor().execute("INSERT INTO mdf_pages (collection_id, doc_id, page, width, height, dpi, fail) VALUES (?, ?, ?, ?, ?, ?, 0)", + (collection_id, doc_id, page, width, height, dpi)) + self.commit() + + def update_page(self, collection_id: int, doc_id: int, page: int, width: int, height: int, dpi: int): + with self.lock: + self.cursor().execute("UPDATE mdf_pages SET width=?, height=?, dpi=?, fail=0 WHERE collection_id=? AND doc_id=? AND page=?", + (width, height, dpi, collection_id, doc_id, page)) + self.commit() + + def add_page_failed(self, collection_id, doc_id, page): + with self.lock: + self.cursor().execute("INSERT INTO mdf_pages (collection_id, doc_id, page, width, height, dpi, fail) VALUES (?, ?, ?, 0, 0, 0, 1)", + (collection_id, doc_id, page)) + self.commit() + + def get_existing_pages(self, fail=0): + cur = self.cursor() + cur.execute("SELECT collection_id, doc_id, page FROM mdf_pages WHERE fail=?", (fail,)) + return cur.fetchall() + + def get_documents(self, range: Optional[tuple[str, str]] = None): + cur = self.cursor() + docs = [] + + sql = "SELECT issue_date, url, pages FROM mdf_links" + if range: + sql += f" WHERE issue_date BETWEEN '{range[0]}' AND '{range[1]}'" + cur.execute(sql) + for issue_date, url, pages in cur.fetchall(): + pub_date, collection_id, doc_id = retronews.parse_url(url) + docs.append(dict( + collection_id=collection_id, + doc_id=doc_id, + pages=pages + )) + + return docs + + def get_doc_pages(self, collection_id, doc_id): + cur = self.cursor() + cur.execute("SELECT page, width, height, dpi FROM mdf_pages WHERE collection_id=? AND doc_id=?", + (collection_id, doc_id)) + return cur.fetchall() + + def fix_documents(self): + cur = self.cursor() + cur.execute("SELECT issue_date, url FROM mdf_links") + for issue_date, url in cur.fetchall(): + pub_date, cid, did = retronews.parse_url(url) + cur.execute("UPDATE mdf_links SET collection_id=?, doc_id=? WHERE issue_date=?", + (cid, did, issue_date)) + self.commit() diff --git a/grab-retronews.py b/grab-retronews.py index fbd1b28..ac4dbf1 100755 --- a/grab-retronews.py +++ b/grab-retronews.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +import logging +import warnings +warnings.filterwarnings("ignore", category=DeprecationWarning) + import os import sys import json @@ -13,6 +17,7 @@ import shutil import queue import traceback import retronews +import logging from database import Database from typing import Optional @@ -20,6 +25,8 @@ from threading import Thread, Lock from time import sleep from argparse import ArgumentParser +warnings.filterwarnings("ignore", category=DeprecationWarning) + VTILES = 3 HTILES = 2 TILE_MERGING_POOL_SIZE = 8 @@ -47,18 +54,27 @@ class DownloaderThread(Thread): _url: str _save_as: str _download_result: Optional[bool] + _handle_http: bool + user_info: dict - def __init__(self, url: str, save_as: str, thread_name=None): + def __init__(self, url: str, save_as: str, thread_name=None, handle_http=False, user_info=None): super().__init__() + if user_info is None: + user_info = {} if thread_name: self.name = thread_name self._url = url self._save_as = save_as self._download_result = None + self._handle_http = handle_http + self.user_info = user_info def run(self): - self._download_result = download_file(self._url, self._save_as) + try: + self._download_result = download_file(self._url, self._save_as, handle_http_errors=not self._handle_http) + except urllib.error.HTTPError: + pass def is_downloaded(self) -> bool: return self._download_result is True @@ -80,7 +96,7 @@ class TileMergeWorker(Thread): try: page = merging_queue.get_nowait() page_dir = os.path.join(self._working_dir, str(page)) - thumbnail_path = os.path.join(self._working_dir, 'thumbnail.jpg') + thumbnail_path = os.path.join(page_dir, 'thumbnail.jpg') meta_path = os.path.join(page_dir, 'meta.json') if os.path.exists(thumbnail_path): @@ -100,12 +116,12 @@ class TileMergeWorker(Thread): for h in range(htiles): vfiles = [] for v in range(vtiles): - vfiles.append(f'v{v}_h{h}.jpg') - run(['convert', '-append', *vfiles, f'_v_{h}.jpg'], cwd=page_dir) - hfiles.append(f'_v_{h}.jpg') + vfiles.append(f'{h}x{v}.jpg') + run(['convert', '-append', *vfiles, f'{h}.jpg'], cwd=page_dir) + hfiles.append(f'{h}.jpg') run(['convert', '+append', *hfiles, os.path.join(self._working_dir, f'{page}.jpg')], cwd=page_dir) - shutil.rmtree(page_dir) + # shutil.rmtree(page_dir) safe_print(f'[tile merger {self._number}] page {page} done') @@ -119,8 +135,9 @@ class PageFetchWorker(Thread): _failed: bool _error: Optional[str] _probe_pages: Optional[list[int]] + _probe_all: bool - def __init__(self, working_dir: str, number: int, collection_id, doc_id, probe_pages: Optional[list[int]] = None): + def __init__(self, working_dir: str, number: int, collection_id, doc_id, probe_pages: Optional[list[int]] = None, probe_all=False): super().__init__() self._working_dir = working_dir self._number = number @@ -129,6 +146,7 @@ class PageFetchWorker(Thread): self._failed = False self._error = None self._probe_pages = probe_pages + self._probe_all = probe_all def run(self): safe_print(f'[pf-{self._number}] started') @@ -140,7 +158,7 @@ class PageFetchWorker(Thread): page = pages_queue.get_nowait() safe_print(f'[pf-{self._number}] page {page} started') - if page in self._probe_pages: + if self._probe_all or page in self._probe_pages: self.probe_dl(page) else: try: @@ -209,28 +227,52 @@ class PageFetchWorker(Thread): real_h = 0 real_v = 0 data_error = False - for h in range(5): - for v in range(5): + dl_tasks = [] + for h in range(10): + for v in range(10): url = retronews.tile_url(self._collection_id, self._doc_id, page, h_tile=h, v_tile=v) - output_file = f'{page_dir}/v{v}_h{h}.jpg' + output_file = f'{page_dir}/{h}x{v}.jpg' if os.path.isfile(output_file): safe_print(f'[pf-{self._number}] probing page {page}: v={v} h={h} ALREADY') if os.path.getsize(output_file) < 4: os.unlink(output_file) continue - try: - if not download_file(url, output_file, handle_http_errors=False): - raise OSError('network failure') - if not imghdr.what(output_file): - data_error = True - break - real_v = v - real_h = h - safe_print(f'[pf-{self._number}] probing page {page}: v={v} h={h} OK') - - except urllib.error.HTTPError: - safe_print(f'[pf-{self._number}] probing page {page}: v={v} h={h} FAIL') - break + + dl_tasks.append(DownloaderThread(url=url, + save_as=os.path.join(page_dir, output_file), + handle_http=True, + thread_name=f'p{page}-v{v}-h{h}', + user_info=dict(h=h, v=v))) + + for task in dl_tasks: + task.start() + for task in dl_tasks: + task.join() + + if task.is_downloaded(): + task_h = task.user_info['h'] + task_v = task.user_info['v'] + if task_h > real_h: + real_h = task_h + if task_v > real_v: + real_v = task_v + + if not imghdr.what(task._save_as): + data_error = True + + # try: + # if not download_file(url, output_file, handle_http_errors=False): + # raise OSError('network failure') + # if not imghdr.what(output_file): + # data_error = True + # break + # real_v = v + # real_h = h + # safe_print(f'[pf-{self._number}] probing page {page}: v={v} h={h} OK') + # + # except urllib.error.HTTPError: + # safe_print(f'[pf-{self._number}] probing page {page}: v={v} h={h} FAIL') + # break if data_error: self.thumbnail_dl(page) @@ -272,10 +314,13 @@ def download_file(url, output, handle_http_errors=True) -> bool: def grab_magazine(url: str, output_root: str, probe_pages: Optional[list[int]] = None, - only_fetch=False, force_overwrite=False): - pub_date, collection_id, doc_id = retronews.parse_url(url) + probe_all=False, only_fetch=False, force_overwrite=False): + try: + pub_date, collection_id, doc_id = retronews.parse_url(url) + except AttributeError: + return False - data = retronews.api_doc_info(collection_id, doc_id) + data = retronews.doc_info(collection_id, doc_id) pages = int(data['nbPages']) print(f'found {pages} pages') @@ -283,7 +328,7 @@ def grab_magazine(url: str, if os.path.exists(os.path.join(output_root, f'{y}-{m}-{d}.pdf')): if not force_overwrite: print(f'{y}-{m}-{d}.pdf already exists, not continuing') - return + return True else: os.unlink(os.path.join(output_root, f'{y}-{m}-{d}.pdf')) print(f'{y}-{m}-{d}.pdf already exists, deleting and continuing (force_overwrite=on)') @@ -302,7 +347,8 @@ def grab_magazine(url: str, number=i+1, collection_id=collection_id, doc_id=doc_id, - probe_pages=probe_pages)) + probe_pages=probe_pages, + probe_all=probe_all)) for worker in pool: worker.start() @@ -312,10 +358,10 @@ def grab_magazine(url: str, with open(os.path.join(output_dir, 'error.txt'), 'w') as f: f.write(f'error: {worker.get_error()}') print(f'ERROR: failed to download {pub_date} magazine') - return + return False if only_fetch: - return + return True # merge tiles for page in range(pages): @@ -338,6 +384,8 @@ def grab_magazine(url: str, except: traceback.print_exc() + return True + if __name__ == '__main__': parser = ArgumentParser() @@ -354,6 +402,8 @@ if __name__ == '__main__': help='only fetch magazine tiles and exit, do not merge anything') parser.add_argument('--force-overwrite', action='store_true', help='if file yyyy-mm-dd.pdf already exists, delete it and start over') + parser.add_argument('--force-probe', action='store_true', + help='force all pages to use the \'probe\' method') parser.add_argument('--fetch-probe-pages', nargs='+', type=int, help='force some pages to use the \'probe\' method, when count of vertical and horizontal tiles is unknown') @@ -371,11 +421,14 @@ if __name__ == '__main__': url = args.url while True: print(f'grabbing {url}...') - grab_magazine(url, - output_root=args.output, - probe_pages=args.fetch_probe_pages, - only_fetch=args.only_fetch, - force_overwrite=args.force_overwrite) + if not grab_magazine(url, + output_root=args.output, + probe_pages=args.fetch_probe_pages, + probe_all=args.force_probe, + only_fetch=args.only_fetch, + force_overwrite=args.force_overwrite): + logging.error('failed to grab') + break if not args.continue_prev and not args.continue_next: break @@ -383,11 +436,16 @@ if __name__ == '__main__': r = requests.get(url) try: + next_url = None if args.continue_next: next_url = re.search(r'SUIVANT', r.text, re.S).groups()[0] elif args.continue_prev: next_url = re.search(r'\s+\s+PRÉCÉDENT', r.text, re.S).groups()[0] + if not next_url: + if not next_url: + break + if next_url.startswith('/'): next_url = f'https://www.retronews.fr{next_url}' diff --git a/retronews.py b/retronews.py deleted file mode 100644 index 6eaf120..0000000 --- a/retronews.py +++ /dev/null @@ -1,48 +0,0 @@ -import re -import requests - -MONTHS = dict( - jan=1, - feb=2, - mar=3, - apr=4, - may=5, - jun=6, - jul=7, - aug=8, - sep=9, - oct=10, - nov=11, - novembre=11, # https://www.retronews.fr/journal/mercure-de-france/15-novembre-1905/118/2617647/1 - dec=12 -) - - -def convert_date(s: str) -> tuple[str, str, str]: - m = re.match(r'^(\d{2})-(.*?)-(\d{4})$', s).groups() - year = m[2] - month = '%02d' % MONTHS[m[1]] - day = m[0] - return year, month, day - - -def parse_url(url: str) -> tuple: - return re.search(r'/(?:mercure-de-france|le-nouveau-mercure|le-mercure-galant|mercure-francais|mercure-galant)/([^/]+)/(\d+)/(\d+)/', url).groups() - - -def doc_info(collection_id, doc_id): - r = requests.get(f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}') - return r.json() - - -def page_info(collection_id, doc_id, page): - r = requests.get(f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/') - return r.json() - - -def thumbnail_url(collection_id, doc_id, page) -> str: - return f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/thumbnail' - - -def tile_url(collection_id, doc_id, page, v_tile, h_tile) -> str: - return f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/tile/{h_tile}/{v_tile}/0' diff --git a/retronews/__init__.py b/retronews/__init__.py new file mode 100644 index 0000000..ae3b518 --- /dev/null +++ b/retronews/__init__.py @@ -0,0 +1,8 @@ +from .retronews import ( + convert_date, + parse_url, + doc_info, + page_info, + thumbnail_url, + tile_url +) \ No newline at end of file diff --git a/retronews/retronews.py b/retronews/retronews.py new file mode 100644 index 0000000..9e80c58 --- /dev/null +++ b/retronews/retronews.py @@ -0,0 +1,50 @@ +import re +import requests + +MONTHS = dict( + jan=1, + feb=2, + mar=3, + apr=4, + may=5, + jun=6, + jul=7, + juillet=7, + aout=8, + aug=8, + sep=9, + oct=10, + nov=11, + novembre=11, # https://www.retronews.fr/journal/mercure-de-france/15-novembre-1905/118/2617647/1 + dec=12 +) + + +def convert_date(s: str) -> tuple[str, str, str]: + m = re.match(r'^(\d{2})-(.*?)-(\d{4})$', s).groups() + year = m[2] + month = '%02d' % MONTHS[m[1]] + day = m[0] + return year, month, day + + +def parse_url(url: str) -> tuple: + return re.search(r'/(?:[\-\d\w]+)/([^/]+)/(\d+)/(\d+)/', url).groups() + + +def doc_info(collection_id, doc_id): + r = requests.get(f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}') + return r.json() + + +def page_info(collection_id, doc_id, page): + r = requests.get(f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/') + return r.json() + + +def thumbnail_url(collection_id, doc_id, page) -> str: + return f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/thumbnail' + + +def tile_url(collection_id, doc_id, page, v_tile, h_tile) -> str: + return f'https://pv5web.retronews.fr/api/document/{collection_id}/{doc_id}/page/{page}/tile/{h_tile}/{v_tile}/0' -- cgit v1.2.3