diff --git a/src/database/questdb_client.py b/src/database/questdb_client.py index 700643b..0d8b0d8 100644 --- a/src/database/questdb_client.py +++ b/src/database/questdb_client.py @@ -1,8 +1,11 @@ import requests import time +import logging from typing import List from ..exchanges.base import Trade +logger = logging.getLogger(__name__) + class DatabaseClient: def __init__(self, host: str = "localhost", port: int = 9000, user: str = None, password: str = None): self.host = host @@ -15,8 +18,8 @@ class DatabaseClient: return total_trades = len(trades) - print(f"Saving {total_trades} trades to QuestDB in batches of {batch_size}...") - + logger.info(f"Speichere {total_trades} Trades in QuestDB (Batches von {batch_size})...") + for i in range(0, total_trades, batch_size): batch = trades[i:i + batch_size] lines = [] @@ -25,34 +28,34 @@ class DatabaseClient: try: symbol = trade.symbol.replace(" ", "\\ ").replace(",", "\\,") exchange = trade.exchange - + line = f"trades,exchange={exchange},symbol={symbol},isin={trade.isin} " \ f"price={trade.price},quantity={trade.quantity} " \ f"{int(trade.timestamp.timestamp() * 1e9)}" lines.append(line) except Exception as e: - print(f"Error formating trade {trade}: {e}") + logger.error(f"Fehler beim Formatieren von Trade {trade}: {e}") continue if not lines: continue payload = "\n".join(lines) + "\n" - + try: response = requests.post( - self.url, - data=payload, + self.url, + data=payload, params={'precision': 'ns'}, auth=self.auth ) if response.status_code not in [204, 200]: - print(f"Error saving batch {i//batch_size + 1} to QuestDB: {response.text}") + logger.error(f"Fehler beim Speichern von Batch {i//batch_size + 1}: {response.text}") else: - print(f"Saved batch {i//batch_size + 1} ({len(batch)} trades)") + logger.info(f"Batch {i//batch_size + 1} gespeichert ({len(batch)} Trades)") except Exception as e: - print(f"Could not connect to QuestDB at {self.url}: {e}") - # Fallback: print to console or save to file + logger.error(f"Verbindung zu QuestDB fehlgeschlagen ({self.url}): {e}") + # Fallback: in Datei speichern self._fallback_save(batch) def _fallback_save(self, trades: List[Trade]): diff --git a/src/exchanges/boersenag.py b/src/exchanges/boersenag.py index 58237f2..ab391e3 100644 --- a/src/exchanges/boersenag.py +++ b/src/exchanges/boersenag.py @@ -8,11 +8,14 @@ URL-Format: https://cld42.boersenag.de/m13data/data/Mifir13DelayedData_{MIC}_{SE import requests import time +import logging from datetime import datetime, timedelta, timezone from typing import List, Optional from .base import BaseExchange, Trade import re +logger = logging.getLogger(__name__) + # Rate-Limiting Konfiguration RATE_LIMIT_DELAY = 0.3 # Sekunden zwischen Requests @@ -176,9 +179,9 @@ class BoersenagBase(BaseExchange): except requests.exceptions.HTTPError as e: if e.response.status_code != 404: - print(f"[{self.name}] HTTP error: {e}") + logger.error(f"[{self.name}] HTTP error: {e}") except Exception as e: - print(f"[{self.name}] Error downloading {url}: {e}") + logger.error(f"[{self.name}] Error downloading {url}: {e}") return trades @@ -265,9 +268,9 @@ class BoersenagBase(BaseExchange): target_date = self._get_last_trading_day(target_date) if target_date != original_date: - print(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") + logger.info(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") - print(f"[{self.name}] Fetching trades for date: {target_date}") + logger.info(f"[{self.name}] Fetching trades for date: {target_date}") # Generiere mögliche URLs urls = self._generate_file_urls(target_date) @@ -281,7 +284,7 @@ class BoersenagBase(BaseExchange): if trades: all_trades.extend(trades) successful += 1 - print(f"[{self.name}] Found {len(trades)} trades from: {url.split('/')[-1]}") + logger.info(f"[{self.name}] Found {len(trades)} trades from: {url.split('/')[-1]}") # Bei Erfolg müssen wir nicht alle anderen URLs probieren break @@ -293,7 +296,7 @@ class BoersenagBase(BaseExchange): if i > 20 and successful == 0: break - print(f"[{self.name}] Total trades fetched: {len(all_trades)}") + logger.info(f"[{self.name}] Total trades fetched: {len(all_trades)}") return all_trades diff --git a/src/exchanges/stuttgart.py b/src/exchanges/stuttgart.py index e9d6207..6c9b57f 100644 --- a/src/exchanges/stuttgart.py +++ b/src/exchanges/stuttgart.py @@ -3,11 +3,14 @@ import gzip import json import csv import io +import logging from datetime import datetime, timedelta, timezone from typing import List, Optional from .base import BaseExchange, Trade from bs4 import BeautifulSoup +logger = logging.getLogger(__name__) + # Browser User-Agent (Vollständiger Browser-Fingerprint für Stuttgart) HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', @@ -84,7 +87,7 @@ class StuttgartExchange(BaseExchange): files = self._generate_expected_urls() except Exception as e: - print(f"[STU] Error fetching page: {e}") + logger.error(f"[STU] Error fetching page: {e}") files = self._generate_expected_urls() return files @@ -194,13 +197,13 @@ class StuttgartExchange(BaseExchange): if trade: trades.append(trade) except Exception as e: - print(f"[STU] Could not parse {url}: {e}") + logger.error(f"[STU] Could not parse {url}: {e}") except requests.exceptions.HTTPError as e: if e.response.status_code != 404: - print(f"[STU] HTTP error downloading {url}: {e}") + logger.error(f"[STU] HTTP error downloading {url}: {e}") except Exception as e: - print(f"[STU] Error downloading {url}: {e}") + logger.error(f"[STU] Error downloading {url}: {e}") return trades @@ -275,7 +278,7 @@ class StuttgartExchange(BaseExchange): ) except Exception as e: - print(f"[STU] Error parsing JSON record: {e}") + logger.debug(f"[STU] Error parsing JSON record: {e}") return None def _parse_csv_row(self, row: dict) -> Optional[Trade]: @@ -331,7 +334,7 @@ class StuttgartExchange(BaseExchange): ) except Exception as e: - print(f"[STU] Error parsing CSV row: {e}") + logger.debug(f"[STU] Error parsing CSV row: {e}") return None def _get_last_trading_day(self, from_date) -> datetime.date: @@ -365,13 +368,13 @@ class StuttgartExchange(BaseExchange): target_date = self._get_last_trading_day(target_date) if target_date != original_date: - print(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") + logger.info(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") - print(f"[{self.name}] Fetching trades for date: {target_date}") + logger.info(f"[{self.name}] Fetching trades for date: {target_date}") # Download-Links holen all_links = self._get_download_links() - print(f"[{self.name}] Found {len(all_links)} potential download links") + logger.info(f"[{self.name}] Found {len(all_links)} potential download links") # Nach Datum filtern target_links = self._filter_files_for_date(all_links, target_date) @@ -380,7 +383,7 @@ class StuttgartExchange(BaseExchange): # Fallback: Versuche alle Links target_links = all_links - print(f"[{self.name}] Trying {len(target_links)} files for target date") + logger.info(f"[{self.name}] Trying {len(target_links)} files for target date") # Dateien herunterladen und parsen successful = 0 @@ -389,9 +392,9 @@ class StuttgartExchange(BaseExchange): if trades: all_trades.extend(trades) successful += 1 - print(f"[{self.name}] Parsed {len(trades)} trades from {url}") + logger.info(f"[{self.name}] Parsed {len(trades)} trades from {url}") - print(f"[{self.name}] Successfully processed {successful} files") - print(f"[{self.name}] Total trades fetched: {len(all_trades)}") + logger.info(f"[{self.name}] Successfully processed {successful} files") + logger.info(f"[{self.name}] Total trades fetched: {len(all_trades)}") return all_trades