diff options
| -rwxr-xr-x | gather.py | 25 |
1 files changed, 15 insertions, 10 deletions
@@ -4,6 +4,7 @@ import logging from time import localtime, mktime, time import requests from base64 import b64decode +from datetime import datetime, timedelta, timezone try: from sqlalchemy import Table, MetaData, Integer, BigInteger, String, Column, Table, ForeignKey, create_engine, select from sqlalchemy.orm import declarative_base, relationship, Session @@ -58,16 +59,14 @@ starting_acsm_id = 177238 guaranteed_large_acsm_id = 1170487 logger.debug(f"created metadata.") -force_acsm_id = 0 -valid_acsms = 0 -only_isbn_acsms = 0 -failed_acsms = 0 -failed_acsms_not200 = 0 -failed_acsms_not200_in_a_row = 0 -hmfan2iarts = 100 # how many failed acsms not 200 in a row to stop - -try: +def update(hmfan2iarts=100): + force_acsm_id = 0 + valid_acsms = 0 + only_isbn_acsms = 0 + failed_acsms = 0 + failed_acsms_not200 = 0 + failed_acsms_not200_in_a_row = 0 with Session(engine) as session: while True: if force_acsm_id != 0: @@ -86,6 +85,9 @@ try: if (r.status_code == 200): failed_acsms_not200_in_a_row = 0 if r.status_code != 200: + if datetime.strptime(borrow.purchase, "%Y-%m-%dT%H:%M:%S%z") > datetime.now(timezone.utc) - timedelta(hours=1): + logger.info(f"we are done for now, as server responded with {r.status_code} for queried acsm id {acsm_id} and the last requested acsm was created less than an hour ago") + break logger.warning(f"received http response with error code not 200 (it is {r.status_code}). if this continues for {hmfan2iarts-failed_acsms_not200_in_a_row} more requests, I'll assume there are no more borrows on the server.") failed_acsms_not200 += 1 failed_acsms_not200_in_a_row += 1 @@ -177,7 +179,10 @@ try: session.add(borrow) session.commit() valid_acsms += 1 + logger.info(f"In this update, {r['valid_acsms']} valid acsms were stored, {r['only_isbn_acsms']} acsms had only isbn and no other data available and {r['failed_acsms']} acsms failed to be received with response code 200 and {r['failed_acsms_not200']} acsms failed to be received but did not return 200. Last valid requested acsm was {r['acsm_id']}. Thank you for cooperation.") + return {"valid_acsms": valid_acsms, "only_isbn_acsms": only_isbn_acsms, "failed_acsms": failed_acsms, "failed_acsms_not200": failed_acsms_not200, "acsm_id": acsm_id} +try: + r = update() except KeyboardInterrupt: logger.warning(f"Keyboard interrupt. Exiting. I hope this terminated cleanly. Last requested acsm was discarded.") -logger.info(f"In this session, {valid_acsms} valid acsms were stored, {only_isbn_acsms} acsms had only isbn and no other data available and {failed_acsms} acsms failed to be received with response code 200 and {failed_acsms_not200} acsms failed to be received but did not return 200. Last valid requested acsm was {acsm_id}. Thank you for cooperation.") |