krumbelfix/trading-daemon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added logging, replaced print calls
All checks were successful
Deployment / deploy-docker (push) Successful in 1m3s
Details

Browse Source
This commit is contained in:
Melchior Reimers
2026-02-16 11:59:53 +01:00
parent 24da2a5861
commit 819b6c2232
3 changed files with 39 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
src/database/questdb_client.py
View File

@@ -1,8 +1,11 @@
import requests import requests
import time import time
import logging
from typing import List from typing import List
from ..exchanges.base import Trade from ..exchanges.base import Trade
logger = logging.getLogger(__name__)
class DatabaseClient: class DatabaseClient:
def __init__(self, host: str = "localhost", port: int = 9000, user: str = None, password: str = None): def __init__(self, host: str = "localhost", port: int = 9000, user: str = None, password: str = None):
self.host = host self.host = host
@@ -15,8 +18,8 @@ class DatabaseClient:
return return
total_trades = len(trades) total_trades = len(trades)
print(f"Saving {total_trades} trades to QuestDB in batches of {batch_size}...") logger.info(f"Speichere {total_trades} Trades in QuestDB (Batches von {batch_size})...")
for i in range(0, total_trades, batch_size): for i in range(0, total_trades, batch_size):
batch = trades[i:i + batch_size] batch = trades[i:i + batch_size]
lines = [] lines = []
@@ -25,34 +28,34 @@ class DatabaseClient:
try: try:
symbol = trade.symbol.replace(" ", "\\ ").replace(",", "\\,") symbol = trade.symbol.replace(" ", "\\ ").replace(",", "\\,")
exchange = trade.exchange exchange = trade.exchange
line = f"trades,exchange={exchange},symbol={symbol},isin={trade.isin} " \ line = f"trades,exchange={exchange},symbol={symbol},isin={trade.isin} " \
f"price={trade.price},quantity={trade.quantity} " \ f"price={trade.price},quantity={trade.quantity} " \
f"{int(trade.timestamp.timestamp() * 1e9)}" f"{int(trade.timestamp.timestamp() * 1e9)}"
lines.append(line) lines.append(line)
except Exception as e: except Exception as e:
print(f"Error formating trade {trade}: {e}") logger.error(f"Fehler beim Formatieren von Trade {trade}: {e}")
continue continue
if not lines: if not lines:
continue continue
payload = "\n".join(lines) + "\n" payload = "\n".join(lines) + "\n"
try: try:
response = requests.post( response = requests.post(
self.url, self.url,
data=payload, data=payload,
params={'precision': 'ns'}, params={'precision': 'ns'},
auth=self.auth auth=self.auth
) )
if response.status_code not in [204, 200]: if response.status_code not in [204, 200]:
print(f"Error saving batch {i//batch_size + 1} to QuestDB: {response.text}") logger.error(f"Fehler beim Speichern von Batch {i//batch_size + 1}: {response.text}")
else: else:
print(f"Saved batch {i//batch_size + 1} ({len(batch)} trades)") logger.info(f"Batch {i//batch_size + 1} gespeichert ({len(batch)} Trades)")
except Exception as e: except Exception as e:
print(f"Could not connect to QuestDB at {self.url}: {e}") logger.error(f"Verbindung zu QuestDB fehlgeschlagen ({self.url}): {e}")
# Fallback: print to console or save to file # Fallback: in Datei speichern
self._fallback_save(batch) self._fallback_save(batch)
def _fallback_save(self, trades: List[Trade]): def _fallback_save(self, trades: List[Trade]):

15
src/exchanges/boersenag.py
View File

@@ -8,11 +8,14 @@ URL-Format: https://cld42.boersenag.de/m13data/data/Mifir13DelayedData_{MIC}_{SE
import requests import requests
import time import time
import logging
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import List, Optional from typing import List, Optional
from .base import BaseExchange, Trade from .base import BaseExchange, Trade
import re import re
logger = logging.getLogger(__name__)
# Rate-Limiting Konfiguration # Rate-Limiting Konfiguration
RATE_LIMIT_DELAY = 0.3 # Sekunden zwischen Requests RATE_LIMIT_DELAY = 0.3 # Sekunden zwischen Requests
@@ -176,9 +179,9 @@ class BoersenagBase(BaseExchange):
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
if e.response.status_code != 404: if e.response.status_code != 404:
print(f"[{self.name}] HTTP error: {e}") logger.error(f"[{self.name}] HTTP error: {e}")
except Exception as e: except Exception as e:
print(f"[{self.name}] Error downloading {url}: {e}") logger.error(f"[{self.name}] Error downloading {url}: {e}")
return trades return trades
@@ -265,9 +268,9 @@ class BoersenagBase(BaseExchange):
target_date = self._get_last_trading_day(target_date) target_date = self._get_last_trading_day(target_date)
if target_date != original_date: if target_date != original_date:
print(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") logger.info(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}")
print(f"[{self.name}] Fetching trades for date: {target_date}") logger.info(f"[{self.name}] Fetching trades for date: {target_date}")
# Generiere mögliche URLs # Generiere mögliche URLs
urls = self._generate_file_urls(target_date) urls = self._generate_file_urls(target_date)
@@ -281,7 +284,7 @@ class BoersenagBase(BaseExchange):
if trades: if trades:
all_trades.extend(trades) all_trades.extend(trades)
successful += 1 successful += 1
print(f"[{self.name}] Found {len(trades)} trades from: {url.split('/')[-1]}") logger.info(f"[{self.name}] Found {len(trades)} trades from: {url.split('/')[-1]}")
# Bei Erfolg müssen wir nicht alle anderen URLs probieren # Bei Erfolg müssen wir nicht alle anderen URLs probieren
break break
@@ -293,7 +296,7 @@ class BoersenagBase(BaseExchange):
if i > 20 and successful == 0: if i > 20 and successful == 0:
break break
print(f"[{self.name}] Total trades fetched: {len(all_trades)}") logger.info(f"[{self.name}] Total trades fetched: {len(all_trades)}")
return all_trades return all_trades

29
src/exchanges/stuttgart.py
View File

@@ -3,11 +3,14 @@ import gzip
import json import json
import csv import csv
import io import io
import logging
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import List, Optional from typing import List, Optional
from .base import BaseExchange, Trade from .base import BaseExchange, Trade
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
logger = logging.getLogger(__name__)
# Browser User-Agent (Vollständiger Browser-Fingerprint für Stuttgart) # Browser User-Agent (Vollständiger Browser-Fingerprint für Stuttgart)
HEADERS = { HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
@@ -84,7 +87,7 @@ class StuttgartExchange(BaseExchange):
files = self._generate_expected_urls() files = self._generate_expected_urls()
except Exception as e: except Exception as e:
print(f"[STU] Error fetching page: {e}") logger.error(f"[STU] Error fetching page: {e}")
files = self._generate_expected_urls() files = self._generate_expected_urls()
return files return files
@@ -194,13 +197,13 @@ class StuttgartExchange(BaseExchange):
if trade: if trade:
trades.append(trade) trades.append(trade)
except Exception as e: except Exception as e:
print(f"[STU] Could not parse {url}: {e}") logger.error(f"[STU] Could not parse {url}: {e}")
except requests.exceptions.HTTPError as e: except requests.exceptions.HTTPError as e:
if e.response.status_code != 404: if e.response.status_code != 404:
print(f"[STU] HTTP error downloading {url}: {e}") logger.error(f"[STU] HTTP error downloading {url}: {e}")
except Exception as e: except Exception as e:
print(f"[STU] Error downloading {url}: {e}") logger.error(f"[STU] Error downloading {url}: {e}")
return trades return trades
@@ -275,7 +278,7 @@ class StuttgartExchange(BaseExchange):
) )
except Exception as e: except Exception as e:
print(f"[STU] Error parsing JSON record: {e}") logger.debug(f"[STU] Error parsing JSON record: {e}")
return None return None
def _parse_csv_row(self, row: dict) -> Optional[Trade]: def _parse_csv_row(self, row: dict) -> Optional[Trade]:
@@ -331,7 +334,7 @@ class StuttgartExchange(BaseExchange):
) )
except Exception as e: except Exception as e:
print(f"[STU] Error parsing CSV row: {e}") logger.debug(f"[STU] Error parsing CSV row: {e}")
return None return None
def _get_last_trading_day(self, from_date) -> datetime.date: def _get_last_trading_day(self, from_date) -> datetime.date:
@@ -365,13 +368,13 @@ class StuttgartExchange(BaseExchange):
target_date = self._get_last_trading_day(target_date) target_date = self._get_last_trading_day(target_date)
if target_date != original_date: if target_date != original_date:
print(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}") logger.info(f"[{self.name}] Skipping weekend: {original_date} -> {target_date}")
print(f"[{self.name}] Fetching trades for date: {target_date}") logger.info(f"[{self.name}] Fetching trades for date: {target_date}")
# Download-Links holen # Download-Links holen
all_links = self._get_download_links() all_links = self._get_download_links()
print(f"[{self.name}] Found {len(all_links)} potential download links") logger.info(f"[{self.name}] Found {len(all_links)} potential download links")
# Nach Datum filtern # Nach Datum filtern
target_links = self._filter_files_for_date(all_links, target_date) target_links = self._filter_files_for_date(all_links, target_date)
@@ -380,7 +383,7 @@ class StuttgartExchange(BaseExchange):
# Fallback: Versuche alle Links # Fallback: Versuche alle Links
target_links = all_links target_links = all_links
print(f"[{self.name}] Trying {len(target_links)} files for target date") logger.info(f"[{self.name}] Trying {len(target_links)} files for target date")
# Dateien herunterladen und parsen # Dateien herunterladen und parsen
successful = 0 successful = 0
@@ -389,9 +392,9 @@ class StuttgartExchange(BaseExchange):
if trades: if trades:
all_trades.extend(trades) all_trades.extend(trades)
successful += 1 successful += 1
print(f"[{self.name}] Parsed {len(trades)} trades from {url}") logger.info(f"[{self.name}] Parsed {len(trades)} trades from {url}")
print(f"[{self.name}] Successfully processed {successful} files") logger.info(f"[{self.name}] Successfully processed {successful} files")
print(f"[{self.name}] Total trades fetched: {len(all_trades)}") logger.info(f"[{self.name}] Total trades fetched: {len(all_trades)}")
return all_trades return all_trades